From 6add7df94ba65f4c69f691e0c6ebc91fa85aacc6 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 9 Aug 2018 09:27:41 +0100 Subject: [PATCH 01/57] [ML] Job and datafeed mappings with index template (#32719) Index mappings for the configuration documents --- .../core/ml/datafeed/DatafeedConfig.java | 1 + .../core/ml/job/config/AnalysisConfig.java | 22 +- .../config/CategorizationAnalyzerConfig.java | 6 +- .../core/ml/job/config/DataDescription.java | 12 +- .../core/ml/job/config/ModelPlotConfig.java | 4 +- .../persistence/AnomalyDetectorsIndex.java | 10 + .../AnomalyDetectorsIndexFields.java | 1 + .../persistence/ElasticsearchMappings.java | 296 +++++++++++++++++- .../ml/job/results/ReservedFieldNames.java | 122 +++++++- .../ElasticsearchMappingsTests.java | 74 +++-- .../xpack/ml/MachineLearning.java | 19 +- .../ml/action/TransportOpenJobAction.java | 2 +- .../job/persistence/JobResultsProvider.java | 2 +- .../MachineLearningLicensingTests.java | 2 +- 14 files changed, 513 insertions(+), 60 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 9cad992327e25..3c6565e13c0ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -63,6 +63,7 @@ public class DatafeedConfig extends AbstractDiffable implements // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("datafeeds"); + public static String TYPE = "datafeed"; /** * The field name used to specify document counts in Elasticsearch diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java index 9068ffda4de55..e4e41697bec62 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java @@ -53,17 +53,17 @@ public class AnalysisConfig implements ToXContentObject, Writeable { * Serialisation names */ public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config"); - private static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); - private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name"); - static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters"); - private static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER; - private static final ParseField LATENCY = new ParseField("latency"); - private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); - private static final ParseField DETECTORS = new ParseField("detectors"); - private static final ParseField INFLUENCERS = new ParseField("influencers"); - private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); - private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); - private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + public static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name"); + public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters"); + public static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER; + public static final ParseField LATENCY = new ParseField("latency"); + public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); + public static final ParseField DETECTORS = new ParseField("detectors"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); + public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); + public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); public static final String ML_CATEGORY_FIELD = "mlcategory"; public static final Set AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java index 36c25e0a7a7aa..e0b66e30f2496 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java @@ -53,9 +53,9 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeable { public static final ParseField CATEGORIZATION_ANALYZER = new ParseField("categorization_analyzer"); - private static final ParseField TOKENIZER = RestAnalyzeAction.Fields.TOKENIZER; - private static final ParseField TOKEN_FILTERS = RestAnalyzeAction.Fields.TOKEN_FILTERS; - private static final ParseField CHAR_FILTERS = RestAnalyzeAction.Fields.CHAR_FILTERS; + public static final ParseField TOKENIZER = RestAnalyzeAction.Fields.TOKENIZER; + public static final ParseField TOKEN_FILTERS = RestAnalyzeAction.Fields.TOKEN_FILTERS; + public static final ParseField CHAR_FILTERS = RestAnalyzeAction.Fields.CHAR_FILTERS; /** * This method is only used in the unit tests - in production code this config is always parsed as a fragment. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java index 87c084baeac95..022181bd8f026 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java @@ -77,12 +77,12 @@ public String toString() { } } - private static final ParseField DATA_DESCRIPTION_FIELD = new ParseField("data_description"); - private static final ParseField FORMAT_FIELD = new ParseField("format"); - private static final ParseField TIME_FIELD_NAME_FIELD = new ParseField("time_field"); - private static final ParseField TIME_FORMAT_FIELD = new ParseField("time_format"); - private static final ParseField FIELD_DELIMITER_FIELD = new ParseField("field_delimiter"); - private static final ParseField QUOTE_CHARACTER_FIELD = new ParseField("quote_character"); + public static final ParseField DATA_DESCRIPTION_FIELD = new ParseField("data_description"); + public static final ParseField FORMAT_FIELD = new ParseField("format"); + public static final ParseField TIME_FIELD_NAME_FIELD = new ParseField("time_field"); + public static final ParseField TIME_FORMAT_FIELD = new ParseField("time_format"); + public static final ParseField FIELD_DELIMITER_FIELD = new ParseField("field_delimiter"); + public static final ParseField QUOTE_CHARACTER_FIELD = new ParseField("quote_character"); /** * Special time format string for epoch times (seconds) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java index 98aa618dd1ee9..824df9f88f5ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java @@ -18,8 +18,8 @@ public class ModelPlotConfig implements ToXContentObject, Writeable { - private static final ParseField TYPE_FIELD = new ParseField("model_plot_config"); - private static final ParseField ENABLED_FIELD = new ParseField("enabled"); + public static final ParseField TYPE_FIELD = new ParseField("model_plot_config"); + public static final ParseField ENABLED_FIELD = new ParseField("enabled"); public static final ParseField TERMS_FIELD = new ParseField("terms"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 4e51d7b6c1e30..6cf4aee2a9672 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -56,4 +56,14 @@ public static String getPhysicalIndexFromState(ClusterState state, String jobId) public static String jobStateIndexName() { return AnomalyDetectorsIndexFields.STATE_INDEX_NAME; } + + /** + * The name of the index where job and datafeed configuration + * is stored + * @return The index name + */ + public static String configIndexName() { + return AnomalyDetectorsIndexFields.CONFIG_INDEX; + } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index 9cdaf10326dfb..527ba5dc1458b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -7,6 +7,7 @@ public final class AnomalyDetectorsIndexFields { + public static final String CONFIG_INDEX = ".ml-config"; public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; public static final String STATE_INDEX_NAME = ".ml-state"; public static final String RESULTS_INDEX_DEFAULT = "shared"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index d95b404ed3571..3d560949963d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -7,8 +7,18 @@ import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; +import org.elasticsearch.xpack.core.ml.job.config.DataDescription; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; +import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; @@ -34,8 +44,8 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; /** - * Static methods to create Elasticsearch mappings for the autodetect - * persisted objects/documents + * Static methods to create Elasticsearch index mappings for the autodetect + * persisted objects/documents and configurations *

* ElasticSearch automatically recognises array types so they are * not explicitly mapped as such. For arrays of objects the type @@ -79,6 +89,11 @@ public class ElasticsearchMappings { */ public static final String ES_DOC = "_doc"; + /** + * The configuration document type + */ + public static final String CONFIG_TYPE = "config_type"; + /** * Elasticsearch data types */ @@ -95,6 +110,277 @@ public class ElasticsearchMappings { private ElasticsearchMappings() { } + public static XContentBuilder configMapping() throws IOException { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject(DOC_TYPE); + addMetaInformation(builder); + addDefaultMapping(builder); + builder.startObject(PROPERTIES); + + addJobConfigFields(builder); + addDatafeedConfigFields(builder); + + builder.endObject() + .endObject() + .endObject(); + return builder; + } + + public static void addJobConfigFields(XContentBuilder builder) throws IOException { + + builder.startObject(CONFIG_TYPE) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.JOB_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.JOB_VERSION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.GROUPS.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.ANALYSIS_CONFIG.getPreferredName()) + .startObject(PROPERTIES) + .startObject(AnalysisConfig.BUCKET_SPAN.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName()) + .startObject(PROPERTIES) + .startObject(CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + // TOKENIZER, TOKEN_FILTERS and CHAR_FILTERS are complex types, don't parse or index + .startObject(CategorizationAnalyzerConfig.TOKENIZER.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName()) + .field(ENABLED, false) + .endObject() + .endObject() + .endObject() + .startObject(AnalysisConfig.LATENCY.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.SUMMARY_COUNT_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.DETECTORS.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(Detector.FUNCTION_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.FIELD_NAME_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.BY_FIELD_NAME_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.OVER_FIELD_NAME_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.USE_NULL_FIELD.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(Detector.EXCLUDE_FREQUENT_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Detector.CUSTOM_RULES_FIELD.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(DetectionRule.ACTIONS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + // RuleScope is a map + .startObject(DetectionRule.SCOPE_FIELD.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(DetectionRule.CONDITIONS_FIELD.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(RuleCondition.APPLIES_TO_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Operator.OPERATOR_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(RuleCondition.VALUE_FIELD.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .startObject(Detector.DETECTOR_INDEX.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .endObject() + .endObject() + + .startObject(AnalysisConfig.INFLUENCERS.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnalysisConfig.OVERLAPPING_BUCKETS.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(AnalysisConfig.RESULT_FINALIZATION_WINDOW.getPreferredName()) + .field(TYPE, LONG) // TODO This should be made a time value + .endObject() + .startObject(AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(AnalysisConfig.USE_PER_PARTITION_NORMALIZATION.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .endObject() + .endObject() + + .startObject(Job.ANALYSIS_LIMITS.getPreferredName()) + .startObject(PROPERTIES) + .startObject(AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName()) + .field(TYPE, KEYWORD) // TODO Should be a ByteSizeValue + .endObject() + .startObject(AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .endObject() + .endObject() + + .startObject(Job.CREATE_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + + .startObject(Job.CUSTOM_SETTINGS.getPreferredName()) + // Custom settings are an untyped map + .field(ENABLED, false) + .endObject() + + .startObject(Job.DATA_DESCRIPTION.getPreferredName()) + .startObject(PROPERTIES) + .startObject(DataDescription.FORMAT_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataDescription.TIME_FIELD_NAME_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataDescription.TIME_FORMAT_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataDescription.FIELD_DELIMITER_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataDescription.QUOTE_CHARACTER_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() + + .startObject(Job.DESCRIPTION.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(Job.FINISHED_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(Job.LAST_DATA_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName()) + .field(TYPE, LONG) // TODO should be ByteSizeValue + .endObject() + + .startObject(Job.MODEL_PLOT_CONFIG.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ModelPlotConfig.ENABLED_FIELD.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(ModelPlotConfig.TERMS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() + + .startObject(Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName()) + .field(TYPE, LONG) // TODO should be TimeValue + .endObject() + .startObject(Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName()) + .field(TYPE, LONG) // TODO should be TimeValue + .endObject() + .startObject(Job.RESULTS_RETENTION_DAYS.getPreferredName()) + .field(TYPE, LONG) // TODO should be TimeValue + .endObject() + .startObject(Job.MODEL_SNAPSHOT_ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.RESULTS_INDEX_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject(); + } + + public static void addDatafeedConfigFields(XContentBuilder builder) throws IOException { + builder.startObject(DatafeedConfig.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DatafeedConfig.QUERY_DELAY.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DatafeedConfig.FREQUENCY.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DatafeedConfig.INDICES.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DatafeedConfig.TYPES.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DatafeedConfig.QUERY.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(DatafeedConfig.SCROLL_SIZE.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DatafeedConfig.AGGREGATIONS.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(DatafeedConfig.SCRIPT_FIELDS.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(DatafeedConfig.CHUNKING_CONFIG.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ChunkingConfig.MODE_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ChunkingConfig.TIME_SPAN_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() + .startObject(DatafeedConfig.HEADERS.getPreferredName()) + .field(ENABLED, false) + .endObject(); + } + /** * Creates a default mapping which has a dynamic template that * treats all dynamically added fields as keywords. This is needed @@ -129,11 +415,11 @@ public static void addMetaInformation(XContentBuilder builder) throws IOExceptio .endObject(); } - public static XContentBuilder docMapping() throws IOException { - return docMapping(Collections.emptyList()); + public static XContentBuilder resultsMapping() throws IOException { + return resultsMapping(Collections.emptyList()); } - public static XContentBuilder docMapping(Collection extraTermFields) throws IOException { + public static XContentBuilder resultsMapping(Collection extraTermFields) throws IOException { XContentBuilder builder = jsonBuilder(); builder.startObject(); builder.startObject(DOC_TYPE); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 8637eb1172281..fd1c6a9d02fb9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -5,8 +5,18 @@ */ package org.elasticsearch.xpack.core.ml.job.results; +import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; +import org.elasticsearch.xpack.core.ml.job.config.DataDescription; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; +import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -36,7 +46,7 @@ public final class ReservedFieldNames { * 2.x requires mappings for given fields be consistent across all types * in a given index.) */ - private static final String[] RESERVED_FIELD_NAME_ARRAY = { + private static final String[] RESERVED_RESULT_FIELD_NAME_ARRAY = { ElasticsearchMappings.ALL_FIELD_VALUES, Job.ID.getPreferredName(), @@ -164,25 +174,121 @@ public final class ReservedFieldNames { }; /** - * Test if fieldName is one of the reserved names or if it contains dots then - * that the segment before the first dot is not a reserved name. A fieldName - * containing dots represents nested fields in which case we only care about - * the top level. + * This array should be updated to contain all the field names that appear + * in any documents we store in our config index. + */ + private static final String[] RESERVED_CONFIG_FIELD_NAME_ARRAY = { + Job.ID.getPreferredName(), + Job.JOB_TYPE.getPreferredName(), + Job.JOB_VERSION.getPreferredName(), + Job.GROUPS.getPreferredName(), + Job.ANALYSIS_CONFIG.getPreferredName(), + Job.ANALYSIS_LIMITS.getPreferredName(), + Job.CREATE_TIME.getPreferredName(), + Job.CUSTOM_SETTINGS.getPreferredName(), + Job.DATA_DESCRIPTION.getPreferredName(), + Job.DESCRIPTION.getPreferredName(), + Job.FINISHED_TIME.getPreferredName(), + Job.LAST_DATA_TIME.getPreferredName(), + Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), + Job.MODEL_PLOT_CONFIG.getPreferredName(), + Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(), + Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName(), + Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), + Job.RESULTS_RETENTION_DAYS.getPreferredName(), + Job.MODEL_SNAPSHOT_ID.getPreferredName(), + Job.RESULTS_INDEX_NAME.getPreferredName(), + + AnalysisConfig.BUCKET_SPAN.getPreferredName(), + AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName(), + AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), + AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName(), + AnalysisConfig.LATENCY.getPreferredName(), + AnalysisConfig.SUMMARY_COUNT_FIELD_NAME.getPreferredName(), + AnalysisConfig.DETECTORS.getPreferredName(), + AnalysisConfig.INFLUENCERS.getPreferredName(), + AnalysisConfig.OVERLAPPING_BUCKETS.getPreferredName(), + AnalysisConfig.RESULT_FINALIZATION_WINDOW.getPreferredName(), + AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName(), + AnalysisConfig.USE_PER_PARTITION_NORMALIZATION.getPreferredName(), + + AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(), + AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), + + CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName(), + CategorizationAnalyzerConfig.TOKENIZER.getPreferredName(), + CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName(), + + Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName(), + Detector.FUNCTION_FIELD.getPreferredName(), + Detector.FIELD_NAME_FIELD.getPreferredName(), + Detector.BY_FIELD_NAME_FIELD.getPreferredName(), + Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), + Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(), + Detector.USE_NULL_FIELD.getPreferredName(), + Detector.EXCLUDE_FREQUENT_FIELD.getPreferredName(), + Detector.CUSTOM_RULES_FIELD.getPreferredName(), + Detector.DETECTOR_INDEX.getPreferredName(), + + DetectionRule.ACTIONS_FIELD.getPreferredName(), + DetectionRule.CONDITIONS_FIELD.getPreferredName(), + DetectionRule.SCOPE_FIELD.getPreferredName(), + RuleCondition.APPLIES_TO_FIELD.getPreferredName(), + RuleCondition.VALUE_FIELD.getPreferredName(), + Operator.OPERATOR_FIELD.getPreferredName(), + + DataDescription.FORMAT_FIELD.getPreferredName(), + DataDescription.TIME_FIELD_NAME_FIELD.getPreferredName(), + DataDescription.TIME_FORMAT_FIELD.getPreferredName(), + DataDescription.FIELD_DELIMITER_FIELD.getPreferredName(), + DataDescription.QUOTE_CHARACTER_FIELD.getPreferredName(), + + ModelPlotConfig.ENABLED_FIELD.getPreferredName(), + ModelPlotConfig.TERMS_FIELD.getPreferredName(), + + DatafeedConfig.ID.getPreferredName(), + DatafeedConfig.QUERY_DELAY.getPreferredName(), + DatafeedConfig.FREQUENCY.getPreferredName(), + DatafeedConfig.INDICES.getPreferredName(), + DatafeedConfig.TYPES.getPreferredName(), + DatafeedConfig.QUERY.getPreferredName(), + DatafeedConfig.SCROLL_SIZE.getPreferredName(), + DatafeedConfig.AGGREGATIONS.getPreferredName(), + DatafeedConfig.SCRIPT_FIELDS.getPreferredName(), + DatafeedConfig.CHUNKING_CONFIG.getPreferredName(), + DatafeedConfig.HEADERS.getPreferredName(), + + ChunkingConfig.MODE_FIELD.getPreferredName(), + ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(), + + ElasticsearchMappings.CONFIG_TYPE + }; + + /** + * Test if fieldName is one of the reserved result fieldnames or if it contains + * dots then that the segment before the first dot is not a reserved results + * fieldname. A fieldName containing dots represents nested fields in which + * case we only care about the top level. * * @param fieldName Document field name. This may contain dots '.' - * @return True if fieldName is not a reserved name or the top level segment + * @return True if fieldName is not a reserved results fieldname or the top level segment * is not a reserved name. */ public static boolean isValidFieldName(String fieldName) { String[] segments = DOT_PATTERN.split(fieldName); - return !RESERVED_FIELD_NAMES.contains(segments[0]); + return RESERVED_RESULT_FIELD_NAMES.contains(segments[0]) == false; } /** * A set of all reserved field names in our results. Fields from the raw * data with these names are not added to any result. */ - public static final Set RESERVED_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_FIELD_NAME_ARRAY)); + public static final Set RESERVED_RESULT_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_RESULT_FIELD_NAME_ARRAY)); + + /** + * A set of all reserved field names in our config. + */ + public static final Set RESERVED_CONFIG_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_CONFIG_FIELD_NAME_ARRAY)); private ReservedFieldNames() { } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index 2b644c4aa5be0..e4ce536a3ccf6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -13,6 +13,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; @@ -28,25 +31,28 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; public class ElasticsearchMappingsTests extends ESTestCase { - public void testReservedFields() throws Exception { - Set overridden = new HashSet<>(); - - // These are not reserved because they're Elasticsearch keywords, not - // field names - overridden.add(ElasticsearchMappings.ANALYZER); - overridden.add(ElasticsearchMappings.COPY_TO); - overridden.add(ElasticsearchMappings.DYNAMIC); - overridden.add(ElasticsearchMappings.ENABLED); - overridden.add(ElasticsearchMappings.NESTED); - overridden.add(ElasticsearchMappings.PROPERTIES); - overridden.add(ElasticsearchMappings.TYPE); - overridden.add(ElasticsearchMappings.WHITESPACE); + // These are not reserved because they're Elasticsearch keywords, not + // field names + private static List KEYWORDS = Arrays.asList( + ElasticsearchMappings.ANALYZER, + ElasticsearchMappings.COPY_TO, + ElasticsearchMappings.DYNAMIC, + ElasticsearchMappings.ENABLED, + ElasticsearchMappings.NESTED, + ElasticsearchMappings.PROPERTIES, + ElasticsearchMappings.TYPE, + ElasticsearchMappings.WHITESPACE + ); + + public void testResultsMapppingReservedFields() throws Exception { + Set overridden = new HashSet<>(KEYWORDS); // These are not reserved because they're data types, not field names overridden.add(Result.TYPE.getPreferredName()); @@ -57,25 +63,44 @@ public void testReservedFields() throws Exception { overridden.add(Quantiles.TYPE.getPreferredName()); Set expected = collectResultsDocFieldNames(); + expected.removeAll(overridden); + + compareFields(expected, ReservedFieldNames.RESERVED_RESULT_FIELD_NAMES); + } + + public void testConfigMapppingReservedFields() throws Exception { + Set overridden = new HashSet<>(KEYWORDS); + + // These are not reserved because they're data types, not field names + overridden.add(Job.TYPE); + overridden.add(DatafeedConfig.TYPE); + // ModelPlotConfig has an 'enabled' the same as one of the keywords + overridden.remove(ModelPlotConfig.ENABLED_FIELD.getPreferredName()); + Set expected = collectConfigDocFieldNames(); expected.removeAll(overridden); - if (ReservedFieldNames.RESERVED_FIELD_NAMES.size() != expected.size()) { - Set diff = new HashSet<>(ReservedFieldNames.RESERVED_FIELD_NAMES); + compareFields(expected, ReservedFieldNames.RESERVED_CONFIG_FIELD_NAMES); + } + + + private void compareFields(Set expected, Set reserved) { + if (reserved.size() != expected.size()) { + Set diff = new HashSet<>(reserved); diff.removeAll(expected); StringBuilder errorMessage = new StringBuilder("Fields in ReservedFieldNames but not in expected: ").append(diff); diff = new HashSet<>(expected); - diff.removeAll(ReservedFieldNames.RESERVED_FIELD_NAMES); + diff.removeAll(reserved); errorMessage.append("\nFields in expected but not in ReservedFieldNames: ").append(diff); fail(errorMessage.toString()); } - assertEquals(ReservedFieldNames.RESERVED_FIELD_NAMES.size(), expected.size()); + assertEquals(reserved.size(), expected.size()); for (String s : expected) { // By comparing like this the failure messages say which string is missing - String reserved = ReservedFieldNames.RESERVED_FIELD_NAMES.contains(s) ? s : null; - assertEquals(s, reserved); + String reservedField = reserved.contains(s) ? s : null; + assertEquals(s, reservedField); } } @@ -105,10 +130,17 @@ public void testTermFieldMapping() throws IOException { private Set collectResultsDocFieldNames() throws IOException { // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. + return collectFieldNames(ElasticsearchMappings.resultsMapping()); + } + + private Set collectConfigDocFieldNames() throws IOException { + // Only the mappings for the config index should be added below. Do NOT add mappings for other indexes here. + return collectFieldNames(ElasticsearchMappings.configMapping()); + } - XContentBuilder builder = ElasticsearchMappings.docMapping(); + private Set collectFieldNames(XContentBuilder mapping) throws IOException { BufferedInputStream inputStream = - new BufferedInputStream(new ByteArrayInputStream(Strings.toString(builder).getBytes(StandardCharsets.UTF_8))); + new BufferedInputStream(new ByteArrayInputStream(Strings.toString(mapping).getBytes(StandardCharsets.UTF_8))); JsonParser parser = new JsonFactory().createParser(inputStream); Set fieldNames = new HashSet<>(); boolean isAfterPropertiesStart = false; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 282c06a37d85c..800995bb25bdf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -644,6 +644,23 @@ public UnaryOperator> getIndexTemplateMetaDat logger.warn("Error loading the template for the " + MlMetaIndex.INDEX_NAME + " index", e); } + try (XContentBuilder configMapping = ElasticsearchMappings.configMapping()) { + IndexTemplateMetaData configTemplate = IndexTemplateMetaData.builder(AnomalyDetectorsIndex.configIndexName()) + .patterns(Collections.singletonList(AnomalyDetectorsIndex.configIndexName())) + .settings(Settings.builder() + // Our indexes are small and one shard puts the + // least possible burden on Elasticsearch + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting)) + .version(Version.CURRENT.id) + .putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(configMapping)) + .build(); + templates.put(AnomalyDetectorsIndex.configIndexName(), configTemplate); + } catch (IOException e) { + logger.warn("Error loading the template for the " + AnomalyDetectorsIndex.configIndexName() + " index", e); + } + try (XContentBuilder stateMapping = ElasticsearchMappings.stateMapping()) { IndexTemplateMetaData stateTemplate = IndexTemplateMetaData.builder(AnomalyDetectorsIndex.jobStateIndexName()) .patterns(Collections.singletonList(AnomalyDetectorsIndex.jobStateIndexName())) @@ -659,7 +676,7 @@ public UnaryOperator> getIndexTemplateMetaDat logger.error("Error loading the template for the " + AnomalyDetectorsIndex.jobStateIndexName() + " index", e); } - try (XContentBuilder docMapping = ElasticsearchMappings.docMapping()) { + try (XContentBuilder docMapping = ElasticsearchMappings.resultsMapping()) { IndexTemplateMetaData jobResultsTemplate = IndexTemplateMetaData.builder(AnomalyDetectorsIndex.jobResultsIndexPrefix()) .patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*")) .settings(Settings.builder() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index b620816cc8252..d688db7fd0d1d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -542,7 +542,7 @@ public void onFailure(Exception e) { ); // Step 1. Try adding results doc mapping - addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobParams.getJobId()), ElasticsearchMappings::docMapping, + addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobParams.getJobId()), ElasticsearchMappings::resultsMapping, state, resultsPutMappingHandler); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 9338d24dd68da..cb00132bfea13 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -336,7 +336,7 @@ public static int countFields(Map mapping) { private void updateIndexMappingWithTermFields(String indexName, Collection termFields, ActionListener listener) { // Put the whole "doc" mapping, not just the term fields, otherwise we'll wipe the _meta section of the mapping - try (XContentBuilder termFieldsMapping = ElasticsearchMappings.docMapping(termFields)) { + try (XContentBuilder termFieldsMapping = ElasticsearchMappings.resultsMapping(termFields)) { final PutMappingRequest request = client.admin().indices().preparePutMapping(indexName).setType(ElasticsearchMappings.DOC_TYPE) .setSource(termFieldsMapping).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request, new ActionListener() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java index e8ac4285b6b33..75e79ede014d4 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java @@ -54,7 +54,7 @@ public void resetLicensing() { ensureYellow(); } - public void testMachineLearningPutJobActionRestricted() throws Exception { + public void testMachineLearningPutJobActionRestricted() { String jobId = "testmachinelearningputjobactionrestricted"; // Pick a license that does not allow machine learning License.OperationMode mode = randomInvalidLicenseType(); From 27a81f7251bcbbcd226c79faa3ea84afb0662b1e Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 13 Aug 2018 14:48:45 +0100 Subject: [PATCH 02/57] [ML] Job config document CRUD operations (#32738) --- .../xpack/core/ml/job/config/Job.java | 23 + .../ml/job/persistence/JobConfigProvider.java | 541 ++++++++++++++++++ .../xpack/ml/MlSingleNodeTestCase.java | 27 + .../AutodetectResultProcessorIT.java | 25 +- .../ml/integration/JobConfigProviderIT.java | 351 ++++++++++++ .../ml/integration/JobResultsProviderIT.java | 30 - .../persistence/JobConfigProviderTests.java | 96 ++++ 7 files changed, 1039 insertions(+), 54 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 5a352ab26657c..de44ee6f7ee5e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -227,6 +227,25 @@ public Job(StreamInput in) throws IOException { deleting = in.readBoolean(); } + /** + * Get the persisted job document name from the Job Id. + * Throws if {@code jobId} is not a valid job Id. + * + * @param jobId The job id + * @return The id of document the job is persisted in + */ + public static String documentId(String jobId) { + if (!MlStrings.isValidId(jobId)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, ID.getPreferredName(), jobId)); + } + if (!MlStrings.hasValidLengthForId(jobId)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_ID_TOO_LONG, MlStrings.ID_LENGTH_LIMIT)); + } + + return "job-" + jobId; + } + + /** * Return the Job Id. * @@ -745,6 +764,10 @@ public void setGroups(List groups) { this.groups = groups == null ? Collections.emptyList() : groups; } + public List getGroups() { + return groups; + } + public Builder setCustomSettings(Map customSettings) { this.customSettings = customSettings; return this; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java new file mode 100644 index 0000000000000..3166ca33c5bb9 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -0,0 +1,541 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +/** + * This class implements CRUD operation for the + * anomaly detector job configuration document + */ +public class JobConfigProvider extends AbstractComponent { + + public static String ALL = "_all"; + + private final Client client; + + public JobConfigProvider(Client client, Settings settings) { + super(settings); + this.client = client; + } + + /** + * Persist the anomaly detector job configuration to the configuration index. + * It is an error if an job with the same Id already exists - the config will + * not be overwritten. + * + * @param job The anomaly detector job configuration + * @param listener Index response listener + */ + public void putJob(Job job, ActionListener listener) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder source = job.toXContent(builder, ToXContent.EMPTY_PARAMS); + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(job.getId())) + .setSource(source) + .setOpType(DocWriteRequest.OpType.CREATE) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + + } catch (IOException e) { + listener.onFailure(new ElasticsearchParseException("Failed to serialise job with id [" + job.getId() + "]", e)); + } + } + + /** + * Get the anomaly detector job specified by {@code jobId}. + * If the job is missing a {@code ResourceNotFoundException} is returned + * via the listener. + * + * @param jobId The job ID + * @param jobListener Job listener + */ + public void getJob(String jobId, ActionListener jobListener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + jobListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + + BytesReference source = getResponse.getSourceAsBytesRef(); + parseJobLenientlyFromSource(source, jobListener); + } + + @Override + public void onFailure(Exception e) { + jobListener.onFailure(e); + } + }); + } + + /** + * Delete the anomaly detector job config document + * + * @param jobId The job id + * @param actionListener Deleted job listener + */ + public void deleteJob(String jobId, ActionListener actionListener) { + DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + + executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + actionListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + + assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED; + actionListener.onResponse(deleteResponse); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + + /** + * Get the job and update it by applying {@code jobUpdater} then index the changed job + * setting the version in the request. Applying the update may cause a validation error + * which is returned via {@code updatedJobListener} + * + * @param jobId The Id of the job to update + * @param update The job update + * @param maxModelMemoryLimit The maximum model memory allowed + * @param updatedJobListener Updated job listener + */ + public void updateJob(String jobId, JobUpdate update, ByteSizeValue maxModelMemoryLimit, ActionListener updatedJobListener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + updatedJobListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + + long version = getResponse.getVersion(); + BytesReference source = getResponse.getSourceAsBytesRef(); + Job.Builder jobBuilder; + try { + jobBuilder = parseJobLenientlyFromSource(source); + } catch (IOException e) { + updatedJobListener.onFailure(new ElasticsearchParseException("failed to parse " + getResponse.getType(), e)); + return; + } + + Job updatedJob; + try { + // Applying the update may result in a validation error + updatedJob = update.mergeWithJob(jobBuilder.build(), maxModelMemoryLimit); + } catch (Exception e) { + updatedJobListener.onFailure(e); + return; + } + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder updatedSource = updatedJob.toXContent(builder, ToXContent.EMPTY_PARAMS); + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(updatedJob.getId())) + .setSource(updatedSource) + .setVersion(version) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + indexResponse -> { + assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; + updatedJobListener.onResponse(updatedJob); + }, + updatedJobListener::onFailure + )); + + } catch (IOException e) { + updatedJobListener.onFailure( + new ElasticsearchParseException("Failed to serialise job with id [" + jobId + "]", e)); + } + + + } + + @Override + public void onFailure(Exception e) { + updatedJobListener.onFailure(e); + } + }); + } + + /** + * Expands an expression into the set of matching names. {@code expresssion} + * may be a wildcard, a job group, a job ID or a list of those. + * If {@code expression} == 'ALL', '*' or the empty string then all + * job IDs are returned. + * Job groups are expanded to all the jobs IDs in that group. + * + * For example, given a set of names ["foo-1", "foo-2", "bar-1", bar-2"], + * expressions resolve follows: + *

    + *
  • "foo-1" : ["foo-1"]
    • + *
  • "bar-1" : ["bar-1"]
    • + *
  • "foo-1,foo-2" : ["foo-1", "foo-2"]
    • + *
  • "foo-*" : ["foo-1", "foo-2"]
    • + *
  • "*-1" : ["bar-1", "foo-1"]
    • + *
  • "*" : ["bar-1", "bar-2", "foo-1", "foo-2"]
    • + *
  • "_all" : ["bar-1", "bar-2", "foo-1", "foo-2"]
    • + *
+ * + * @param expression the expression to resolve + * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. + * This only applies to wild card expressions, if {@code expression} is not a + * wildcard then setting this true will not suppress the exception + * @param listener The expanded job IDs listener + */ + public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener> listener) { + String [] tokens = tokenizeExpression(expression); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + sourceBuilder.sort(Job.ID.getPreferredName()); + String [] includes = new String[] {Job.ID.getPreferredName(), Job.GROUPS.getPreferredName()}; + sourceBuilder.fetchSource(includes, null); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + LinkedList requiredMatches = requiredMatches(tokens, allowNoJobs); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + Set jobIds = new HashSet<>(); + Set groupsIds = new HashSet<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + jobIds.add((String)hit.getSourceAsMap().get(Job.ID.getPreferredName())); + List groups = (List)hit.getSourceAsMap().get(Job.GROUPS.getPreferredName()); + if (groups != null) { + groupsIds.addAll(groups); + } + } + + groupsIds.addAll(jobIds); + filterMatchedIds(requiredMatches, groupsIds); + if (requiredMatches.isEmpty() == false) { + // some required jobs were not found + String missing = requiredMatches.stream().map(IdMatcher::getId).collect(Collectors.joining(",")); + listener.onFailure(ExceptionsHelper.missingJobException(missing)); + return; + } + + listener.onResponse(jobIds); + }, + listener::onFailure) + , client::search); + + } + + /** + * The same logic as {@link #expandJobsIds(String, boolean, ActionListener)} but + * the full anomaly detector job configuration is returned. + * + * See {@link #expandJobsIds(String, boolean, ActionListener)} + * + * @param expression the expression to resolve + * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. + * This only applies to wild card expressions, if {@code expression} is not a + * wildcard then setting this true will not suppress the exception + * @param listener The expanded jobs listener + */ + // NORELEASE jobs should be paged or have a mechanism to return all jobs if there are many of them + public void expandJobs(String expression, boolean allowNoJobs, ActionListener> listener) { + String [] tokens = tokenizeExpression(expression); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + sourceBuilder.sort(Job.ID.getPreferredName()); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + LinkedList requiredMatches = requiredMatches(tokens, allowNoJobs); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + List jobs = new ArrayList<>(); + Set jobAndGroupIds = new HashSet<>(); + + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + try { + BytesReference source = hit.getSourceRef(); + Job.Builder job = parseJobLenientlyFromSource(source); + jobs.add(job); + jobAndGroupIds.add(job.getId()); + jobAndGroupIds.addAll(job.getGroups()); + } catch (IOException e) { + // TODO A better way to handle this rather than just ignoring the error? + logger.error("Error parsing anomaly detector job configuration [" + hit.getId() + "]", e); + } + } + + filterMatchedIds(requiredMatches, jobAndGroupIds); + if (requiredMatches.isEmpty() == false) { + // some required jobs were not found + String missing = requiredMatches.stream().map(IdMatcher::getId).collect(Collectors.joining(",")); + listener.onFailure(ExceptionsHelper.missingJobException(missing)); + return; + } + + listener.onResponse(jobs); + }, + listener::onFailure) + , client::search); + + } + + private void parseJobLenientlyFromSource(BytesReference source, ActionListener jobListener) { + try (InputStream stream = source.streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + jobListener.onResponse(Job.LENIENT_PARSER.apply(parser, null)); + } catch (Exception e) { + jobListener.onFailure(e); + } + } + + private Job.Builder parseJobLenientlyFromSource(BytesReference source) throws IOException { + try (InputStream stream = source.streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + return Job.LENIENT_PARSER.apply(parser, null); + } + } + + private QueryBuilder buildQuery(String [] tokens) { + QueryBuilder jobQuery = new TermQueryBuilder(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE); + if (isWildcardAll(tokens)) { + // match all + return jobQuery; + } + + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(jobQuery); + BoolQueryBuilder shouldQueries = new BoolQueryBuilder(); + + List terms = new ArrayList<>(); + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token)) { + shouldQueries.should(new WildcardQueryBuilder(Job.ID.getPreferredName(), token)); + shouldQueries.should(new WildcardQueryBuilder(Job.GROUPS.getPreferredName(), token)); + } else { + terms.add(token); + } + } + + if (terms.isEmpty() == false) { + shouldQueries.should(new TermsQueryBuilder(Job.ID.getPreferredName(), terms)); + shouldQueries.should(new TermsQueryBuilder(Job.GROUPS.getPreferredName(), terms)); + } + + if (shouldQueries.should().isEmpty() == false) { + boolQueryBuilder.filter(shouldQueries); + } + + return boolQueryBuilder; + } + + /** + * Does the {@code tokens} array resolves to a wildcard all expression. + * True if {@code tokens} is empty or if it contains a single element + * equal to {@link #ALL}, '*' or an empty string + * + * @param tokens Expression tokens + * @return True if tokens resolves to a wildcard all expression + */ + static boolean isWildcardAll(String [] tokens) { + if (tokens.length == 0) { + return true; + } + return tokens.length == 1 && (ALL.equals(tokens[0]) || Regex.isMatchAllPattern(tokens[0]) || tokens[0].isEmpty()); + } + + static String [] tokenizeExpression(String expression) { + return Strings.tokenizeToStringArray(expression, ","); + } + + /** + * Generate the list of required matches from the expressions in {@code tokens} + * + * @param tokens List of expressions that may be wildcards or full Ids + * @param allowNoJobForWildcards If true then it is not required for wildcard + * expressions to match an Id meaning they are + * not returned in the list of required matches + * @return A list of required Id matchers + */ + static LinkedList requiredMatches(String [] tokens, boolean allowNoJobForWildcards) { + LinkedList matchers = new LinkedList<>(); + + if (isWildcardAll(tokens)) { + // if allowNoJobForWildcards == true then any number + // of jobs with any id is ok. Therefore no matches + // are required + + if (allowNoJobForWildcards == false) { + // require something, anything to match + matchers.add(new WildcardMatcher("*")); + } + return matchers; + } + + if (allowNoJobForWildcards) { + // matches are not required for wildcards but + // specific job Ids are + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token) == false) { + matchers.add(new EqualsIdMatcher(token)); + } + } + } else { + // Matches are required for wildcards + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token)) { + matchers.add(new WildcardMatcher(token)); + } else { + matchers.add(new EqualsIdMatcher(token)); + } + } + } + + return matchers; + } + + /** + * For each given {@code requiredMatchers} check there is an element + * present in {@code ids} that matches. Once a match is made the + * matcher is popped from {@code requiredMatchers}. + * + * If all matchers are satisfied the list {@code requiredMatchers} will + * be empty after the call otherwise only the unmatched remain. + * + * @param requiredMatchers This is modified by the function: all matched matchers + * are removed from the list. At the end of the call only + * the unmatched ones are in this list + * @param ids Ids required to be matched + */ + static void filterMatchedIds(LinkedList requiredMatchers, Collection ids) { + for (String id: ids) { + Iterator itr = requiredMatchers.iterator(); + if (itr.hasNext() == false) { + break; + } + while (itr.hasNext()) { + if (itr.next().matches(id)) { + itr.remove(); + } + } + } + } + + abstract static class IdMatcher { + protected final String id; + + IdMatcher(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public abstract boolean matches(String jobId); + } + + static class EqualsIdMatcher extends IdMatcher { + EqualsIdMatcher(String id) { + super(id); + } + + @Override + public boolean matches(String id) { + return this.id.equals(id); + } + } + + static class WildcardMatcher extends IdMatcher { + WildcardMatcher(String id) { + super(id); + } + + @Override + public boolean matches(String id) { + return Regex.simpleMatch(this.id, id); + } + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java index 60f08067a9bcf..0668b29c626ed 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java @@ -5,11 +5,17 @@ */ package org.elasticsearch.xpack.ml; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.license.LicenseService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.MachineLearningField; +import java.util.Collection; + /** * An extension to {@link ESSingleNodeTestCase} that adds node settings specifically needed for ML test cases. */ @@ -18,10 +24,31 @@ public abstract class MlSingleNodeTestCase extends ESSingleNodeTestCase { @Override protected Settings nodeSettings() { Settings.Builder newSettings = Settings.builder(); + newSettings.put(super.nodeSettings()); + // Disable native ML autodetect_process as the c++ controller won't be available newSettings.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); + newSettings.put(MachineLearningField.MAX_MODEL_MEMORY_LIMIT.getKey(), new ByteSizeValue(1024)); newSettings.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); + // Disable security otherwise delete-by-query action fails to get authorized + newSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); + newSettings.put(XPackSettings.MONITORING_ENABLED.getKey(), false); + newSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); return newSettings.build(); } + @Override + protected Collection> getPlugins() { + return pluginList(LocalStateMachineLearning.class); + } + + protected void waitForMlTemplates() throws Exception { + // block until the templates are installed + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + assertTrue("Timed out waiting for the ML templates to be installed", + MachineLearning.allTemplatesInstalled(state)); + }); + } + } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index 756eeb8626dc6..84e09735c214a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; @@ -14,7 +13,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; @@ -33,7 +31,6 @@ import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.ml.LocalStateMachineLearning; -import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder; @@ -78,17 +75,6 @@ public class AutodetectResultProcessorIT extends MlSingleNodeTestCase { private AutoDetectResultProcessor resultProcessor; private Renormalizer renormalizer; - @Override - protected Settings nodeSettings() { - Settings.Builder newSettings = Settings.builder(); - newSettings.put(super.nodeSettings()); - // Disable security otherwise delete-by-query action fails to get authorized - newSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); - newSettings.put(XPackSettings.MONITORING_ENABLED.getKey(), false); - newSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); - return newSettings.build(); - } - @Override protected Collection> getPlugins() { return pluginList(LocalStateMachineLearning.class, ReindexPlugin.class); @@ -109,7 +95,7 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { capturedUpdateModelSnapshotOnJobRequests.add(modelSnapshot); } }; - putIndexTemplates(); + waitForMlTemplates(); putJob(); } @@ -288,15 +274,6 @@ public void testEndOfStreamTriggersPersisting() throws Exception { assertResultsAreSame(allRecords, persistedRecords); } - private void putIndexTemplates() throws Exception { - // block until the templates are installed - assertBusy(() -> { - ClusterState state = client().admin().cluster().prepareState().get().getState(); - assertTrue("Timed out waiting for the ML templates to be installed", - MachineLearning.allTemplatesInstalled(state)); - }); - } - private void putJob() { Detector detector = new Detector.Builder("dc", "by_instance").build(); Job.Builder jobBuilder = new Job.Builder(JOB_ID); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java new file mode 100644 index 0000000000000..fb82b1c74d0eb --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -0,0 +1,351 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.DataDescription; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.core.ml.job.config.Detector; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.config.RuleScope; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.core.IsInstanceOf.instanceOf; + +public class JobConfigProviderIT extends MlSingleNodeTestCase { + + private JobConfigProvider jobConfigProvider; + + @Before + public void createComponents() throws Exception { + jobConfigProvider = new JobConfigProvider(client(), Settings.EMPTY); + waitForMlTemplates(); + } + + public void testGetMissingJob() throws InterruptedException { + AtomicReference jobHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(actionListener -> jobConfigProvider.getJob("missing", actionListener), jobHolder, exceptionHolder); + + assertNull(jobHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + } + + public void testOverwriteNotAllowed() throws InterruptedException { + final String jobId = "same-id"; + + AtomicReference indexResponseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + // Create job + Job initialJob = createJob(jobId, null).build(new Date()); + blockingCall(actionListener -> jobConfigProvider.putJob(initialJob, actionListener), indexResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertNotNull(indexResponseHolder.get()); + + indexResponseHolder.set(null); + Job jobWithSameId = createJob(jobId, null).build(new Date()); + blockingCall(actionListener -> jobConfigProvider.putJob(jobWithSameId, actionListener), indexResponseHolder, exceptionHolder); + assertNull(indexResponseHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(VersionConflictEngineException.class)); + } + + public void testCrud() throws InterruptedException { + final String jobId = "crud-job"; + + AtomicReference indexResponseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + // Create job + Job newJob = createJob(jobId, null).build(new Date()); + blockingCall(actionListener -> jobConfigProvider.putJob(newJob, actionListener), indexResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertNotNull(indexResponseHolder.get()); + + // Read Job + AtomicReference getJobResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.getJob(jobId, actionListener), getJobResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + + assertEquals(newJob, getJobResponseHolder.get().build()); + + // Update Job + indexResponseHolder.set(null); + JobUpdate jobUpdate = new JobUpdate.Builder(jobId).setDescription("This job has been updated").build(); + + AtomicReference updateJobResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.updateJob(jobId, jobUpdate, new ByteSizeValue(32), actionListener), + updateJobResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals("This job has been updated", updateJobResponseHolder.get().getDescription()); + + getJobResponseHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.getJob(jobId, actionListener), getJobResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals("This job has been updated", getJobResponseHolder.get().build().getDescription()); + + // Delete Job + AtomicReference deleteJobResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, actionListener), + deleteJobResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(deleteJobResponseHolder.get().getResult(), equalTo(DocWriteResponse.Result.DELETED)); + + // Read deleted job + getJobResponseHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.getJob(jobId, actionListener), getJobResponseHolder, exceptionHolder); + assertNull(getJobResponseHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + + // Delete deleted job + deleteJobResponseHolder.set(null); + exceptionHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, actionListener), + deleteJobResponseHolder, exceptionHolder); + assertNull(deleteJobResponseHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + } + + public void testUpdateWithAValidationError() throws Exception { + final String jobId = "bad-update-job"; + + AtomicReference indexResponseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + // Create job + Job newJob = createJob(jobId, null).build(new Date()); + blockingCall(actionListener -> jobConfigProvider.putJob(newJob, actionListener), indexResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertNotNull(indexResponseHolder.get()); + + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().exclude("not a used field", "filerfoo")).build(); + JobUpdate.DetectorUpdate detectorUpdate = new JobUpdate.DetectorUpdate(0, null, Collections.singletonList(rule)); + JobUpdate invalidUpdate = new JobUpdate.Builder(jobId) + .setDetectorUpdates(Collections.singletonList(detectorUpdate)) + .build(); + + AtomicReference updateJobResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.updateJob(jobId, invalidUpdate, new ByteSizeValue(32), actionListener), + updateJobResponseHolder, exceptionHolder); + assertNull(updateJobResponseHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ElasticsearchStatusException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("Invalid detector rule:")); + } + + public void testAllowNoJobs() throws InterruptedException { + AtomicReference> jobIdsHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", false, actionListener), + jobIdsHolder, exceptionHolder); + + assertNull(jobIdsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); + + exceptionHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, actionListener), + jobIdsHolder, exceptionHolder); + assertNotNull(jobIdsHolder.get()); + assertNull(exceptionHolder.get()); + + AtomicReference> jobsHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", false, actionListener), + jobsHolder, exceptionHolder); + + assertNull(jobsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); + + exceptionHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, actionListener), + jobsHolder, exceptionHolder); + assertNotNull(jobsHolder.get()); + assertNull(exceptionHolder.get()); + } + + public void testExpandJobs_GroupsAndJobIds() throws Exception { + Job tom = putJob(createJob("tom", null)); + Job dick = putJob(createJob("dick", null)); + Job harry = putJob(createJob("harry", Collections.singletonList("harry-group"))); + Job harryJnr = putJob(createJob("harry-jnr", Collections.singletonList("harry-group"))); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + // Job Ids + Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("tom", "dick", "harry", "harry-jnr")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("tom", "dick", "harry", "harry-jnr")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,harry", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("tom", "harry")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("harry-group,tom", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("harry", "harry-jnr", "tom")), expandedIds); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference> jobIdsHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,missing1,missing2", true, actionListener), + jobIdsHolder, exceptionHolder); + assertNull(jobIdsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), equalTo("No known job with id 'missing1,missing2'")); + + // Job builders + List expandedJobsBuilders = blockingCall(actionListener -> + jobConfigProvider.expandJobs("harry-group,tom", false, actionListener)); + List expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(harry, harryJnr, tom)); + + expandedJobsBuilders = blockingCall(actionListener -> + jobConfigProvider.expandJobs("_all", false, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(tom, dick, harry, harryJnr)); + + expandedJobsBuilders = blockingCall(actionListener -> + jobConfigProvider.expandJobs("tom,harry", false, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(tom, harry)); + + expandedJobsBuilders = blockingCall(actionListener -> + jobConfigProvider.expandJobs("", false, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(tom, dick, harry, harryJnr)); + + AtomicReference> jobsHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.expandJobs("tom,missing1,missing2", false, actionListener), + jobsHolder, exceptionHolder); + assertNull(jobsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), equalTo("No known job with id 'missing1,missing2'")); + } + + public void testExpandJobs_WildCardExpansion() throws Exception { + Job foo1 = putJob(createJob("foo-1", null)); + Job foo2 = putJob(createJob("foo-2", null)); + Job bar1 = putJob(createJob("bar-1", Collections.singletonList("bar"))); + Job bar2 = putJob(createJob("bar-2", Collections.singletonList("bar"))); + Job nbar = putJob(createJob("nbar", Collections.singletonList("bar"))); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + // Test job IDs only + Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*-1", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("bar-1", "foo-1")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("bar*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("bar-1", "bar-2", "nbar")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("b*r-1", true, actionListener)); + assertEquals(new TreeSet<>(Collections.singletonList("bar-1")), expandedIds); + + // Test full job config + List expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("foo*", true, actionListener)); + List expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(foo1, foo2)); + + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("*-1", true, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(foo1, bar1)); + + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("bar*", true, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(bar1, bar2, nbar)); + + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("b*r-1", true, actionListener)); + expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); + assertThat(expandedJobs, containsInAnyOrder(bar1)); + } + + private Job.Builder createJob(String jobId, List groups) { + Detector.Builder d1 = new Detector.Builder("info_content", "domain"); + d1.setOverFieldName("client"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d1.build())); + + Job.Builder builder = new Job.Builder(); + builder.setId(jobId); + builder.setAnalysisConfig(ac); + builder.setDataDescription(new DataDescription.Builder()); + if (groups != null && groups.isEmpty() == false) { + builder.setGroups(groups); + } + return builder; + } + + private void blockingCall(Consumer> function, AtomicReference response, + AtomicReference error) throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + ActionListener listener = ActionListener.wrap( + r -> { + response.set(r); + latch.countDown(); + }, + e -> { + error.set(e); + latch.countDown(); + } + ); + + function.accept(listener); + latch.await(); + } + + private T blockingCall(Consumer> function) throws Exception { + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + blockingCall(function, responseHolder, exceptionHolder); + if (exceptionHolder.get() != null) { + assertNotNull(exceptionHolder.get().getMessage(), exceptionHolder.get()); + } + return responseHolder.get(); + } + + private Job putJob(Job.Builder job) throws Exception { + Job builtJob = job.build(new Date()); + this.blockingCall(actionListener -> jobConfigProvider.putJob(builtJob, actionListener)); + return builtJob; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index e36c313b626c9..09651f554d848 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -18,8 +17,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PutJobAction; @@ -40,8 +37,6 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; -import org.elasticsearch.xpack.ml.LocalStateMachineLearning; -import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; @@ -55,7 +50,6 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; @@ -76,21 +70,6 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase { private JobResultsProvider jobProvider; - @Override - protected Settings nodeSettings() { - Settings.Builder newSettings = Settings.builder(); - newSettings.put(super.nodeSettings()); - newSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); - newSettings.put(XPackSettings.MONITORING_ENABLED.getKey(), false); - newSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); - return newSettings.build(); - } - - @Override - protected Collection> getPlugins() { - return pluginList(LocalStateMachineLearning.class); - } - @Before public void createComponents() throws Exception { Settings.Builder builder = Settings.builder() @@ -99,15 +78,6 @@ public void createComponents() throws Exception { waitForMlTemplates(); } - private void waitForMlTemplates() throws Exception { - // block until the templates are installed - assertBusy(() -> { - ClusterState state = client().admin().cluster().prepareState().get().getState(); - assertTrue("Timed out waiting for the ML templates to be installed", - MachineLearning.allTemplatesInstalled(state)); - }); - } - public void testGetCalandarByJobId() throws Exception { List calendars = new ArrayList<>(); calendars.add(new Calendar("empty calendar", Collections.emptyList(), null)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java new file mode 100644 index 0000000000000..04bcd57e64fc0 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isOneOf; + +public class JobConfigProviderTests extends ESTestCase { + + public void testMatchingJobIds() { + LinkedList requiredMatches = JobConfigProvider.requiredMatches(new String[] {"*"}, false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(""), false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(null), false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(null), false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); + assertThat(requiredMatches, hasSize(1)); + assertThat(requiredMatches.get(0).getId(), equalTo("*")); + + requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression("_all"), false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*"}, false); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","foo2")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches, hasSize(2)); + JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","foo2")); + assertThat(requiredMatches, hasSize(1)); + assertEquals("bar", requiredMatches.get(0).getId()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches, hasSize(2)); + JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","bar")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches, hasSize(2)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("bar")); + assertThat(requiredMatches, hasSize(1)); + assertEquals("foo*", requiredMatches.get(0).getId()); + + requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression("foo,bar,baz,wild*"), false); + assertThat(requiredMatches, hasSize(4)); + JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo","baz")); + assertThat(requiredMatches, hasSize(2)); + assertThat(requiredMatches.get(0).getId(), isOneOf("bar", "wild*")); + assertThat(requiredMatches.get(1).getId(), isOneOf("bar", "wild*")); + } + + public void testMatchingJobIds_allowNoJobs() { + // wildcard all with allow no jobs + LinkedList requiredMatches = JobConfigProvider.requiredMatches(new String[] {"*"}, true); + assertThat(requiredMatches, empty()); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, true); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("bar")); + assertThat(requiredMatches, empty()); + + requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, true); + assertThat(requiredMatches, hasSize(1)); + JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); + assertThat(requiredMatches, hasSize(1)); + assertEquals("bar", requiredMatches.get(0).getId()); + } +} From 59a1205157ec7d14b25fd4aa4ecb90a6e5e4b528 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 16 Aug 2018 10:09:32 +0100 Subject: [PATCH 03/57] [ML] Datafeed config CRUD operations (#32854) --- .../xpack/core/ml/MlMetaIndex.java | 2 - .../xpack/core/ml/MlMetadata.java | 2 +- .../core/ml/action/PutDatafeedAction.java | 4 +- .../xpack/core/ml/calendars/Calendar.java | 4 +- .../core/ml/calendars/ScheduledEvent.java | 4 +- .../core/ml/datafeed/DatafeedConfig.java | 32 +- .../xpack/core/ml/job/config/Detector.java | 2 +- .../xpack/core/ml/job/config/MlFilter.java | 4 +- .../xpack/core/ml/utils/ToXContentParams.java | 12 +- .../core/ml/datafeed/DatafeedConfigTests.java | 42 +- .../TransportPostCalendarEventsAction.java | 3 +- .../ml/action/TransportPutCalendarAction.java | 3 +- .../ml/action/TransportPutFilterAction.java | 3 +- .../action/TransportUpdateFilterAction.java | 3 +- .../persistence/DatafeedConfigProvider.java | 393 ++++++++++++++++++ .../job/persistence/ExpandedIdsMatcher.java | 158 +++++++ .../ml/job/persistence/JobConfigProvider.java | 163 +------- .../xpack/ml/MlSingleNodeTestCase.java | 32 ++ .../integration/DatafeedConfigProviderIT.java | 253 +++++++++++ .../ml/integration/JobConfigProviderIT.java | 31 -- .../ml/integration/JobResultsProviderIT.java | 7 +- .../persistence/ExpandedIdsMatcherTests.java | 101 +++++ .../persistence/JobConfigProviderTests.java | 96 ----- 23 files changed, 1047 insertions(+), 307 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcher.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcherTests.java delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index d625e6e311aaf..9014c415f16bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -21,8 +21,6 @@ public final class MlMetaIndex { */ public static final String INDEX_NAME = ".ml-meta"; - public static final String INCLUDE_TYPE_KEY = "include_type"; - public static final String TYPE = "doc"; private MlMetaIndex() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 8d3c6a3565f93..14736a764390b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -167,7 +167,7 @@ private static void writeMap(Map map, StreamOut @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { DelegatingMapParams extendedParams = - new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_CLUSTER_STATE, "true"), params); + new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"), params); mapValuesToXContent(JOBS_FIELD, jobs, builder, extendedParams); mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, extendedParams); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java index 4d3f720026e14..448d826973595 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java @@ -138,9 +138,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - datafeed.doXContentBody(builder, params); - builder.endObject(); + datafeed.toXContent(builder, params); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/Calendar.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/Calendar.java index 9add81aace357..723f1b5c8b7ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/Calendar.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/Calendar.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.MlMetaIndex; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; import java.util.Arrays; @@ -111,7 +111,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } - if (params.paramAsBoolean(MlMetaIndex.INCLUDE_TYPE_KEY, false)) { + if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false)) { builder.field(TYPE.getPreferredName(), CALENDAR_TYPE); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java index 79e569987fa02..042775c8024e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleAction; @@ -23,6 +22,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.Intervals; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils; import java.io.IOException; @@ -170,7 +170,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (eventId != null) { builder.field(EVENT_ID.getPreferredName(), eventId); } - if (params.paramAsBoolean(MlMetaIndex.INCLUDE_TYPE_KEY, false)) { + if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false)) { builder.field(TYPE.getPreferredName(), SCHEDULED_EVENT_TYPE); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 3c6565e13c0ff..21faf0a3456cc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -72,6 +72,7 @@ public class DatafeedConfig extends AbstractDiffable implements public static final String DOC_COUNT = "doc_count"; public static final ParseField ID = new ParseField("datafeed_id"); + public static final ParseField CONFIG_TYPE = new ParseField("config_type"); public static final ParseField QUERY_DELAY = new ParseField("query_delay"); public static final ParseField FREQUENCY = new ParseField("frequency"); public static final ParseField INDEXES = new ParseField("indexes"); @@ -94,6 +95,7 @@ private static ObjectParser createParser(boolean ignoreUnknownFie ObjectParser parser = new ObjectParser<>("datafeed_config", ignoreUnknownFields, Builder::new); parser.declareString(Builder::setId, ID); + parser.declareString((c, s) -> {}, CONFIG_TYPE); parser.declareString(Builder::setJobId, Job.ID); parser.declareStringArray(Builder::setIndices, INDEXES); parser.declareStringArray(Builder::setIndices, INDICES); @@ -199,6 +201,16 @@ public DatafeedConfig(StreamInput in) throws IOException { } } + /** + * The name of datafeed configuration document name from the datafeed ID. + * + * @param datafeedId The datafeed ID + * @return The ID of document the datafeed config is persisted in + */ + public static String documentId(String datafeedId) { + return "datafeed-" + datafeedId; + } + public String getId() { return id; } @@ -207,6 +219,10 @@ public String getJobId() { return jobId; } + public String getConfigType() { + return TYPE; + } + public TimeValue getQueryDelay() { return queryDelay; } @@ -297,14 +313,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - doXContentBody(builder, params); - builder.endObject(); - return builder; - } - - public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field(ID.getPreferredName(), id); builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false) == true) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); @@ -326,9 +339,10 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (chunkingConfig != null) { builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); } - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_CLUSTER_STATE, false) == true) { + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false) == true) { builder.field(HEADERS.getPreferredName(), headers); } + builder.endObject(); return builder; } @@ -468,6 +482,10 @@ public void setId(String datafeedId) { id = ExceptionsHelper.requireNonNull(datafeedId, ID.getPreferredName()); } + public String getId() { + return id; + } + public void setJobId(String jobId) { this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java index d53e4cb74126d..a09bc376da2b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java @@ -302,7 +302,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // negative means "unknown", which should only happen for a 5.4 job if (detectorIndex >= 0 // no point writing this to cluster state, as the indexes will get reassigned on reload anyway - && params.paramAsBoolean(ToXContentParams.FOR_CLUSTER_STATE, false) == false) { + && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false) == false) { builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java index 48051fa4733ff..f2be3315b4dc7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java @@ -14,10 +14,10 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlStrings; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; import java.util.Arrays; @@ -101,7 +101,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DESCRIPTION.getPreferredName(), description); } builder.field(ITEMS.getPreferredName(), items); - if (params.paramAsBoolean(MlMetaIndex.INCLUDE_TYPE_KEY, false)) { + if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false)) { builder.field(TYPE.getPreferredName(), FILTER_TYPE); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index d120e8cf6685e..f7fb9d46ec8a8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -12,9 +12,17 @@ public final class ToXContentParams { /** - * Parameter to indicate whether we are serialising to X Content for cluster state output. + * Parameter to indicate whether we are serialising to X Content for + * internal storage. Certain fields need to be persisted but should + * not be visible everywhere. */ - public static final String FOR_CLUSTER_STATE = "for_cluster_state"; + public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + + /** + * When serialising POJOs to X Content this indicates whether the type field + * should be included or not + */ + public static final String INCLUDE_TYPE = "include_type"; private ToXContentParams() { } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index 36bd2fbcb4689..f0c7806fd8cf9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -8,13 +8,17 @@ import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -36,17 +40,22 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig.Mode; import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TimeZone; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; @@ -63,6 +72,10 @@ public static DatafeedConfig createRandomizedDatafeedConfig(String jobId) { } public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, long bucketSpanMillis) { + return createRandomizedDatafeedConfigBuilder(jobId, bucketSpanMillis).build(); + } + + private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(String jobId, long bucketSpanMillis) { DatafeedConfig.Builder builder = new DatafeedConfig.Builder(randomValidDatafeedId(), jobId); builder.setIndices(randomStringList(1, 10)); builder.setTypes(randomStringList(0, 10)); @@ -109,7 +122,7 @@ public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, long b if (randomBoolean()) { builder.setChunkingConfig(ChunkingConfigTests.createRandomizedChunk()); } - return builder.build(); + return builder; } @Override @@ -167,6 +180,33 @@ public void testFutureMetadataParse() throws IOException { assertNotNull(DatafeedConfig.LENIENT_PARSER.apply(parser, null).build()); } + public void testToXContentForInternalStorage() throws IOException { + DatafeedConfig.Builder builder = createRandomizedDatafeedConfigBuilder("foo", 300); + + // headers are only persisted to cluster state + Map headers = new HashMap<>(); + headers.put("header-name", "header-value"); + builder.setHeaders(headers); + DatafeedConfig config = builder.build(); + + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")); + + BytesReference forClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, params, false); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, forClusterstateXContent.streamInput()); + + DatafeedConfig parsedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build(); + assertThat(parsedConfig.getHeaders(), hasEntry("header-name", "header-value")); + + // headers are not written without the FOR_INTERNAL_STORAGE param + BytesReference nonClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false); + parser = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, nonClusterstateXContent.streamInput()); + + parsedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build(); + assertThat(parsedConfig.getHeaders().entrySet(), hasSize(0)); + } + public void testCopyConstructor() { for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { DatafeedConfig datafeedConfig = createTestInstance(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java index 7284a490eaa8f..24bc4ad016e7b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.calendars.Calendar; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; @@ -67,7 +68,7 @@ protected void doExecute(Task task, PostCalendarEventsAction.Request request, IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { indexRequest.source(event.toXContent(builder, - new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, + new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")))); } catch (IOException e) { throw new IllegalStateException("Failed to serialise event", e); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java index 7611a27cd5a1d..d50b798ebe729 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.action.PutCalendarAction; import org.elasticsearch.xpack.core.ml.calendars.Calendar; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; import java.util.Collections; @@ -53,7 +54,7 @@ protected void doExecute(Task task, PutCalendarAction.Request request, ActionLis IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { indexRequest.source(calendar.toXContent(builder, - new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")))); + new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")))); } catch (IOException e) { throw new IllegalStateException("Failed to serialise calendar with id [" + calendar.getId() + "]", e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java index 19bf35aaed617..0414e1cdf140c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.ml.action.PutFilterAction; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; import java.util.Collections; @@ -53,7 +54,7 @@ protected void doExecute(Task task, PutFilterAction.Request request, ActionListe indexRequest.opType(DocWriteRequest.OpType.CREATE); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")); indexRequest.source(filter.toXContent(builder, params)); } catch (IOException e) { throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java index c8dbf9273829f..abbefa1e4936f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.job.JobManager; import java.io.IOException; @@ -105,7 +106,7 @@ private void indexUpdatedFilter(MlFilter filter, long version, UpdateFilterActio indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")); indexRequest.source(filter.toXContent(builder, params)); } catch (IOException e) { throw new IllegalStateException("Failed to serialise filter with id [" + filter.getId() + "]", e); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java new file mode 100644 index 0000000000000..9702f1096ecf4 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; +import org.elasticsearch.xpack.ml.job.persistence.ExpandedIdsMatcher; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public class DatafeedConfigProvider extends AbstractComponent { + + private final Client client; + private final NamedXContentRegistry xContentRegistry; + + private static final Map TO_XCONTENT_PARAMS = new HashMap<>(); + static { + TO_XCONTENT_PARAMS.put(ToXContentParams.FOR_INTERNAL_STORAGE, "true"); + TO_XCONTENT_PARAMS.put(ToXContentParams.INCLUDE_TYPE, "true"); + } + + public DatafeedConfigProvider(Client client, Settings settings, NamedXContentRegistry xContentRegistry) { + super(settings); + this.client = client; + this.xContentRegistry = xContentRegistry; + } + + /** + * Persist the datafeed configuration to the config index. + * It is an error if a datafeed with the same Id already exists - + * the config will not be overwritten. + * + * @param config The datafeed configuration + * @param listener Index response listener + */ + public void putDatafeedConfig(DatafeedConfig config, ActionListener listener) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder source = config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); + + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(config.getId())) + .setSource(source) + .setOpType(DocWriteRequest.OpType.CREATE) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + + } catch (IOException e) { + listener.onFailure(new ElasticsearchParseException("Failed to serialise datafeed config with id [" + config.getId() + "]", e)); + } + } + + /** + * Get the datafeed config specified by {@code datafeedId}. + * If the datafeed document is missing a {@code ResourceNotFoundException} + * is returned via the listener. + * + * @param datafeedId The datafeed ID + * @param datafeedConfigListener The config listener + */ + public void getDatafeedConfig(String datafeedId, ActionListener datafeedConfigListener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)); + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + datafeedConfigListener.onFailure(ExceptionsHelper.missingDatafeedException(datafeedId)); + return; + } + BytesReference source = getResponse.getSourceAsBytesRef(); + parseLenientlyFromSource(source, datafeedConfigListener); + } + @Override + public void onFailure(Exception e) { + datafeedConfigListener.onFailure(e); + } + }); + } + + /** + * Delete the datafeed config document + * + * @param datafeedId The datafeed id + * @param actionListener Deleted datafeed listener + */ + public void deleteDatafeedConfig(String datafeedId, ActionListener actionListener) { + DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)); + executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + actionListener.onFailure(ExceptionsHelper.missingDatafeedException(datafeedId)); + return; + } + assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED; + actionListener.onResponse(deleteResponse); + } + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + + /** + * Get the datafeed config and apply the {@code update} + * then index the modified config setting the version in the request. + * + * @param datafeedId The Id of the datafeed to update + * @param update The update + * @param headers Datafeed headers applied with the update + * @param updatedConfigListener Updated datafeed config listener + */ + public void updateDatefeedConfig(String datafeedId, DatafeedUpdate update, Map headers, + ActionListener updatedConfigListener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)); + + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + updatedConfigListener.onFailure(ExceptionsHelper.missingDatafeedException(datafeedId)); + return; + } + long version = getResponse.getVersion(); + BytesReference source = getResponse.getSourceAsBytesRef(); + DatafeedConfig.Builder configBuilder; + try { + configBuilder = parseLenientlyFromSource(source); + } catch (IOException e) { + updatedConfigListener.onFailure( + new ElasticsearchParseException("Failed to parse datafeed config [" + datafeedId + "]", e)); + return; + } + + DatafeedConfig updatedConfig; + try { + updatedConfig = update.apply(configBuilder.build(), headers); + } catch (Exception e) { + updatedConfigListener.onFailure(e); + return; + } + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder updatedSource = updatedConfig.toXContent(builder, ToXContent.EMPTY_PARAMS); + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(updatedConfig.getId())) + .setSource(updatedSource) + .setVersion(version) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + indexResponse -> { + assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; + updatedConfigListener.onResponse(updatedConfig); + }, + updatedConfigListener::onFailure + )); + + } catch (IOException e) { + updatedConfigListener.onFailure( + new ElasticsearchParseException("Failed to serialise datafeed config with id [" + datafeedId + "]", e)); + } + } + + @Override + public void onFailure(Exception e) { + updatedConfigListener.onFailure(e); + } + }); + } + + /** + * Expands an expression into the set of matching names. {@code expresssion} + * may be a wildcard, a datafeed ID or a list of those. + * If {@code expression} == 'ALL', '*' or the empty string then all + * datafeed IDs are returned. + * + * For example, given a set of names ["foo-1", "foo-2", "bar-1", bar-2"], + * expressions resolve follows: + *
    + *
  • "foo-1" : ["foo-1"]
    • + *
  • "bar-1" : ["bar-1"]
    • + *
  • "foo-1,foo-2" : ["foo-1", "foo-2"]
    • + *
  • "foo-*" : ["foo-1", "foo-2"]
    • + *
  • "*-1" : ["bar-1", "foo-1"]
    • + *
  • "*" : ["bar-1", "bar-2", "foo-1", "foo-2"]
    • + *
  • "_all" : ["bar-1", "bar-2", "foo-1", "foo-2"]
    • + *
+ * + * @param expression the expression to resolve + * @param allowNoDatafeeds if {@code false}, an error is thrown when no name matches the {@code expression}. + * This only applies to wild card expressions, if {@code expression} is not a + * wildcard then setting this true will not suppress the exception + * @param listener The expanded datafeed IDs listener + */ + public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, ActionListener> listener) { + String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); + String [] includes = new String[] {DatafeedConfig.ID.getPreferredName()}; + sourceBuilder.fetchSource(includes, null); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + Set datafeedIds = new HashSet<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + datafeedIds.add((String)hit.getSourceAsMap().get(DatafeedConfig.ID.getPreferredName())); + } + + requiredMatches.filterMatchedIds(datafeedIds); + if (requiredMatches.hasUnmatchedIds()) { + // some required datafeeds were not found + listener.onFailure(ExceptionsHelper.missingDatafeedException(requiredMatches.unmatchedIdsString())); + return; + } + + listener.onResponse(datafeedIds); + }, + listener::onFailure) + , client::search); + + } + + /** + * The same logic as {@link #expandDatafeedIds(String, boolean, ActionListener)} but + * the full datafeed configuration is returned. + * + * See {@link #expandDatafeedIds(String, boolean, ActionListener)} + * + * @param expression the expression to resolve + * @param allowNoDatafeeds if {@code false}, an error is thrown when no name matches the {@code expression}. + * This only applies to wild card expressions, if {@code expression} is not a + * wildcard then setting this true will not suppress the exception + * @param listener The expanded datafeed config listener + */ + // NORELEASE datafeed configs should be paged or have a mechanism to return all jobs if there are many of them + public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, ActionListener> listener) { + String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + List datafeeds = new ArrayList<>(); + Set datafeedIds = new HashSet<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + try { + BytesReference source = hit.getSourceRef(); + DatafeedConfig.Builder datafeed = parseLenientlyFromSource(source); + datafeeds.add(datafeed); + datafeedIds.add(datafeed.getId()); + } catch (IOException e) { + // TODO A better way to handle this rather than just ignoring the error? + logger.error("Error parsing datafeed configuration [" + hit.getId() + "]", e); + } + } + + requiredMatches.filterMatchedIds(datafeedIds); + if (requiredMatches.hasUnmatchedIds()) { + // some required datafeeds were not found + listener.onFailure(ExceptionsHelper.missingDatafeedException(requiredMatches.unmatchedIdsString())); + return; + } + + listener.onResponse(datafeeds); + }, + listener::onFailure) + , client::search); + + } + + private QueryBuilder buildQuery(String [] tokens) { + QueryBuilder jobQuery = new TermQueryBuilder(DatafeedConfig.CONFIG_TYPE.getPreferredName(), DatafeedConfig.TYPE); + if (Strings.isAllOrWildcard(tokens)) { + // match all + return jobQuery; + } + + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(jobQuery); + BoolQueryBuilder shouldQueries = new BoolQueryBuilder(); + + List terms = new ArrayList<>(); + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token)) { + shouldQueries.should(new WildcardQueryBuilder(DatafeedConfig.ID.getPreferredName(), token)); + } else { + terms.add(token); + } + } + + if (terms.isEmpty() == false) { + shouldQueries.should(new TermsQueryBuilder(DatafeedConfig.ID.getPreferredName(), terms)); + } + + if (shouldQueries.should().isEmpty() == false) { + boolQueryBuilder.filter(shouldQueries); + } + + return boolQueryBuilder; + } + + private void parseLenientlyFromSource(BytesReference source, ActionListener datafeedConfigListener) { + try (InputStream stream = source.streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream)) { + datafeedConfigListener.onResponse(DatafeedConfig.LENIENT_PARSER.apply(parser, null)); + } catch (Exception e) { + datafeedConfigListener.onFailure(e); + } + } + + private DatafeedConfig.Builder parseLenientlyFromSource(BytesReference source) throws IOException { + try (InputStream stream = source.streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream)) { + return DatafeedConfig.LENIENT_PARSER.apply(parser, null); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcher.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcher.java new file mode 100644 index 0000000000000..4f4968a9d5629 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcher.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; + +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Class for tracking the set of Ids returned from some + * function a satisfy the required Ids as defined by an + * expression that may contain wildcards. + * + * For example, given a set of Ids ["foo-1", "foo-2", "bar-1", bar-2"]: + *
    + *
  • The expression foo* would be satisfied by foo-1 and foo-2
    • + *
  • The expression bar-1 would be satisfied by bar-1
    • + *
  • The expression bar-1,car-1 would leave car-1 unmatched
    • + *
  • The expression * would be satisfied by anything or nothing depending on the + * value of {@code allowNoMatchForWildcards}
    • + *
+ */ +public final class ExpandedIdsMatcher { + + public static String ALL = "_all"; + + /** + * Split {@code expression} into tokens separated by a ',' + * + * @param expression Expression containing zero or more ','s + * @return Array of tokens + */ + public static String [] tokenizeExpression(String expression) { + return Strings.tokenizeToStringArray(expression, ","); + } + + private final LinkedList requiredMatches; + + /** + * Generate the list of required matches from the expressions in {@code tokens} + * and initialize. + * + * @param tokens List of expressions that may be wildcards or full Ids + * @param allowNoMatchForWildcards If true then it is not required for wildcard + * expressions to match an Id meaning they are + * not returned in the list of required matches + */ + public ExpandedIdsMatcher(String [] tokens, boolean allowNoMatchForWildcards) { + requiredMatches = new LinkedList<>(); + + if (Strings.isAllOrWildcard(tokens)) { + // if allowNoJobForWildcards == true then any number + // of jobs with any id is ok. Therefore no matches + // are required + + if (allowNoMatchForWildcards == false) { + // require something, anything to match + requiredMatches.add(new WildcardMatcher("*")); + } + return; + } + + if (allowNoMatchForWildcards) { + // matches are not required for wildcards but + // specific job Ids are + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token) == false) { + requiredMatches.add(new EqualsIdMatcher(token)); + } + } + } else { + // Matches are required for wildcards + for (String token : tokens) { + if (Regex.isSimpleMatchPattern(token)) { + requiredMatches.add(new WildcardMatcher(token)); + } else { + requiredMatches.add(new EqualsIdMatcher(token)); + } + } + } + } + + /** + * For each {@code requiredMatchers} check there is an element + * present in {@code ids} that matches. Once a match is made the + * matcher is removed from {@code requiredMatchers}. + */ + public void filterMatchedIds(Collection ids) { + for (String id: ids) { + Iterator itr = requiredMatches.iterator(); + if (itr.hasNext() == false) { + break; + } + while (itr.hasNext()) { + if (itr.next().matches(id)) { + itr.remove(); + } + } + } + } + + public boolean hasUnmatchedIds() { + return requiredMatches.isEmpty() == false; + } + + public List unmatchedIds() { + return requiredMatches.stream().map(IdMatcher::getId).collect(Collectors.toList()); + } + + public String unmatchedIdsString() { + return requiredMatches.stream().map(IdMatcher::getId).collect(Collectors.joining(",")); + } + + + private abstract static class IdMatcher { + protected final String id; + + IdMatcher(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public abstract boolean matches(String jobId); + } + + private static class EqualsIdMatcher extends IdMatcher { + EqualsIdMatcher(String id) { + super(id); + } + + @Override + public boolean matches(String id) { + return this.id.equals(id); + } + } + + private static class WildcardMatcher extends IdMatcher { + WildcardMatcher(String id) { + super(id); + } + + @Override + public boolean matches(String id) { + return Regex.simpleMatch(this.id, id); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 3166ca33c5bb9..1b89ecb1250ce 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -51,13 +51,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -68,8 +64,6 @@ */ public class JobConfigProvider extends AbstractComponent { - public static String ALL = "_all"; - private final Client client; public JobConfigProvider(Client client, Settings settings) { @@ -189,7 +183,8 @@ public void onResponse(GetResponse getResponse) { try { jobBuilder = parseJobLenientlyFromSource(source); } catch (IOException e) { - updatedJobListener.onFailure(new ElasticsearchParseException("failed to parse " + getResponse.getType(), e)); + updatedJobListener.onFailure( + new ElasticsearchParseException("Failed to parse job configuration [" + jobId + "]", e)); return; } @@ -222,8 +217,6 @@ public void onResponse(GetResponse getResponse) { updatedJobListener.onFailure( new ElasticsearchParseException("Failed to serialise job with id [" + jobId + "]", e)); } - - } @Override @@ -259,7 +252,7 @@ public void onFailure(Exception e) { * @param listener The expanded job IDs listener */ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener> listener) { - String [] tokens = tokenizeExpression(expression); + String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); sourceBuilder.sort(Job.ID.getPreferredName()); String [] includes = new String[] {Job.ID.getPreferredName(), Job.GROUPS.getPreferredName()}; @@ -269,7 +262,7 @@ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder).request(); - LinkedList requiredMatches = requiredMatches(tokens, allowNoJobs); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( @@ -286,11 +279,10 @@ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener } groupsIds.addAll(jobIds); - filterMatchedIds(requiredMatches, groupsIds); - if (requiredMatches.isEmpty() == false) { + requiredMatches.filterMatchedIds(groupsIds); + if (requiredMatches.hasUnmatchedIds()) { // some required jobs were not found - String missing = requiredMatches.stream().map(IdMatcher::getId).collect(Collectors.joining(",")); - listener.onFailure(ExceptionsHelper.missingJobException(missing)); + listener.onFailure(ExceptionsHelper.missingJobException(requiredMatches.unmatchedIdsString())); return; } @@ -315,7 +307,7 @@ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener */ // NORELEASE jobs should be paged or have a mechanism to return all jobs if there are many of them public void expandJobs(String expression, boolean allowNoJobs, ActionListener> listener) { - String [] tokens = tokenizeExpression(expression); + String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); sourceBuilder.sort(Job.ID.getPreferredName()); @@ -323,7 +315,7 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener
  • requiredMatches = requiredMatches(tokens, allowNoJobs); + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( @@ -345,11 +337,10 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener
  • requiredMatches(String [] tokens, boolean allowNoJobForWildcards) { - LinkedList matchers = new LinkedList<>(); - - if (isWildcardAll(tokens)) { - // if allowNoJobForWildcards == true then any number - // of jobs with any id is ok. Therefore no matches - // are required - - if (allowNoJobForWildcards == false) { - // require something, anything to match - matchers.add(new WildcardMatcher("*")); - } - return matchers; - } - - if (allowNoJobForWildcards) { - // matches are not required for wildcards but - // specific job Ids are - for (String token : tokens) { - if (Regex.isSimpleMatchPattern(token) == false) { - matchers.add(new EqualsIdMatcher(token)); - } - } - } else { - // Matches are required for wildcards - for (String token : tokens) { - if (Regex.isSimpleMatchPattern(token)) { - matchers.add(new WildcardMatcher(token)); - } else { - matchers.add(new EqualsIdMatcher(token)); - } - } - } - - return matchers; - } - - /** - * For each given {@code requiredMatchers} check there is an element - * present in {@code ids} that matches. Once a match is made the - * matcher is popped from {@code requiredMatchers}. - * - * If all matchers are satisfied the list {@code requiredMatchers} will - * be empty after the call otherwise only the unmatched remain. - * - * @param requiredMatchers This is modified by the function: all matched matchers - * are removed from the list. At the end of the call only - * the unmatched ones are in this list - * @param ids Ids required to be matched - */ - static void filterMatchedIds(LinkedList requiredMatchers, Collection ids) { - for (String id: ids) { - Iterator itr = requiredMatchers.iterator(); - if (itr.hasNext() == false) { - break; - } - while (itr.hasNext()) { - if (itr.next().matches(id)) { - itr.remove(); - } - } - } - } - - abstract static class IdMatcher { - protected final String id; - - IdMatcher(String id) { - this.id = id; - } - - public String getId() { - return id; - } - - public abstract boolean matches(String jobId); - } - - static class EqualsIdMatcher extends IdMatcher { - EqualsIdMatcher(String id) { - super(id); - } - - @Override - public boolean matches(String id) { - return this.id.equals(id); - } - } - - static class WildcardMatcher extends IdMatcher { - WildcardMatcher(String id) { - super(id); - } - - @Override - public boolean matches(String id) { - return Regex.simpleMatch(this.id, id); - } - } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java index 0668b29c626ed..181636de13663 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -15,6 +16,9 @@ import org.elasticsearch.xpack.core.ml.MachineLearningField; import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; /** * An extension to {@link ESSingleNodeTestCase} that adds node settings specifically needed for ML test cases. @@ -51,4 +55,32 @@ protected void waitForMlTemplates() throws Exception { }); } + protected void blockingCall(Consumer> function, AtomicReference response, + AtomicReference error) throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + ActionListener listener = ActionListener.wrap( + r -> { + response.set(r); + latch.countDown(); + }, + e -> { + error.set(e); + latch.countDown(); + } + ); + + function.accept(listener); + latch.await(); + } + + protected T blockingCall(Consumer> function) throws Exception { + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + blockingCall(function, responseHolder, exceptionHolder); + if (exceptionHolder.get() != null) { + assertNotNull(exceptionHolder.get().getMessage(), exceptionHolder.get()); + } + return responseHolder.get(); + } + } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java new file mode 100644 index 0000000000000..8eeeb2908cf88 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class DatafeedConfigProviderIT extends MlSingleNodeTestCase { + + private DatafeedConfigProvider datafeedConfigProvider; + + @Before + public void createComponents() throws Exception { + datafeedConfigProvider = new DatafeedConfigProvider(client(), Settings.EMPTY, xContentRegistry()); + waitForMlTemplates(); + } + + public void testCrud() throws InterruptedException { + String datafeedId = "df1"; + + AtomicReference indexResponseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + // Create datafeed config + DatafeedConfig config = createDatafeedConfig(datafeedId, "j1"); + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + indexResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals(RestStatus.CREATED, indexResponseHolder.get().status()); + + // Read datafeed config + AtomicReference configBuilderHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.getDatafeedConfig(datafeedId, actionListener), + configBuilderHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals(config, configBuilderHolder.get().build()); + + // Update + DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedId); + List updateIndices = Collections.singletonList("a-different-index"); + update.setIndices(updateIndices); + Map updateHeaders = new HashMap<>(); + // Only security headers are updated, grab the first one + String securityHeader = ClientHelper.SECURITY_HEADER_FILTERS.iterator().next(); + updateHeaders.put(securityHeader, "CHANGED"); + + AtomicReference configHolder = new AtomicReference<>(); + blockingCall(actionListener -> + datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), updateHeaders, actionListener), + configHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(configHolder.get().getIndices(), equalTo(updateIndices)); + assertThat(configHolder.get().getHeaders().get(securityHeader), equalTo("CHANGED")); + + // Delete + AtomicReference deleteResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.deleteDatafeedConfig(datafeedId, actionListener), + deleteResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals(DocWriteResponse.Result.DELETED, deleteResponseHolder.get().getResult()); + } + + public void testMultipleCreateAndDeletes() throws InterruptedException { + String datafeedId = "df2"; + + AtomicReference indexResponseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + // Create datafeed config + DatafeedConfig config = createDatafeedConfig(datafeedId, "j1"); + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + indexResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals(RestStatus.CREATED, indexResponseHolder.get().status()); + + // cannot create another with the same id + indexResponseHolder.set(null); + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + indexResponseHolder, exceptionHolder); + assertNull(indexResponseHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(VersionConflictEngineException.class)); + + // delete + exceptionHolder.set(null); + AtomicReference deleteResponseHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.deleteDatafeedConfig(datafeedId, actionListener), + deleteResponseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertEquals(DocWriteResponse.Result.DELETED, deleteResponseHolder.get().getResult()); + + // error deleting twice + deleteResponseHolder.set(null); + blockingCall(actionListener -> datafeedConfigProvider.deleteDatafeedConfig(datafeedId, actionListener), + deleteResponseHolder, exceptionHolder); + assertNull(deleteResponseHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + } + + public void testUpdateWithAValidationError() throws Exception { + final String datafeedId = "df-bad-update"; + + DatafeedConfig config = createDatafeedConfig(datafeedId, "j2"); + putDatafeedConfig(config); + + DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedId); + update.setId("wrong-datafeed-id"); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference configHolder = new AtomicReference<>(); + blockingCall(actionListener -> + datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), Collections.emptyMap(), actionListener), + configHolder, exceptionHolder); + assertNull(configHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(IllegalArgumentException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("Cannot apply update to datafeedConfig with different id")); + } + + public void testAllowNoDatafeeds() throws InterruptedException { + AtomicReference> datafeedIdsHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("_all", false, actionListener), + datafeedIdsHolder, exceptionHolder); + + assertNull(datafeedIdsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("No datafeed with id [*] exists")); + + exceptionHolder.set(null); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("_all", true, actionListener), + datafeedIdsHolder, exceptionHolder); + assertNotNull(datafeedIdsHolder.get()); + assertNull(exceptionHolder.get()); + + AtomicReference> datafeedsHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", false, actionListener), + datafeedsHolder, exceptionHolder); + + assertNull(datafeedsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(ResourceNotFoundException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("No datafeed with id [*] exists")); + + exceptionHolder.set(null); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", true, actionListener), + datafeedsHolder, exceptionHolder); + assertNotNull(datafeedsHolder.get()); + assertNull(exceptionHolder.get()); + } + + public void testExpandDatafeeds() throws Exception { + DatafeedConfig foo1 = putDatafeedConfig(createDatafeedConfig("foo-1", "j1")); + DatafeedConfig foo2 = putDatafeedConfig(createDatafeedConfig("foo-2", "j2")); + DatafeedConfig bar1 = putDatafeedConfig(createDatafeedConfig("bar-1", "j3")); + DatafeedConfig bar2 = putDatafeedConfig(createDatafeedConfig("bar-2", "j4")); + putDatafeedConfig(createDatafeedConfig("not-used", "j5")); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + // Test job IDs only + Set expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("foo*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); + + expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("*-1", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("bar-1", "foo-1")), expandedIds); + + expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("bar*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("bar-1", "bar-2")), expandedIds); + + expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("b*r-1", true, actionListener)); + assertEquals(new TreeSet<>(Collections.singletonList("bar-1")), expandedIds); + + expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("bar-1,foo*", true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("bar-1", "foo-1", "foo-2")), expandedIds); + + // Test full job config + List expandedDatafeedBuilders = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("foo*", true, actionListener)); + List expandedDatafeeds = + expandedDatafeedBuilders.stream().map(DatafeedConfig.Builder::build).collect(Collectors.toList()); + assertThat(expandedDatafeeds, containsInAnyOrder(foo1, foo2)); + + expandedDatafeedBuilders = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*-1", true, actionListener)); + expandedDatafeeds = expandedDatafeedBuilders.stream().map(DatafeedConfig.Builder::build).collect(Collectors.toList()); + assertThat(expandedDatafeeds, containsInAnyOrder(foo1, bar1)); + + expandedDatafeedBuilders = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("bar*", true, actionListener)); + expandedDatafeeds = expandedDatafeedBuilders.stream().map(DatafeedConfig.Builder::build).collect(Collectors.toList()); + assertThat(expandedDatafeeds, containsInAnyOrder(bar1, bar2)); + + expandedDatafeedBuilders = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("b*r-1", true, actionListener)); + expandedDatafeeds = expandedDatafeedBuilders.stream().map(DatafeedConfig.Builder::build).collect(Collectors.toList()); + assertThat(expandedDatafeeds, containsInAnyOrder(bar1)); + + expandedDatafeedBuilders = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("bar-1,foo*", true, actionListener)); + expandedDatafeeds = expandedDatafeedBuilders.stream().map(DatafeedConfig.Builder::build).collect(Collectors.toList()); + assertThat(expandedDatafeeds, containsInAnyOrder(bar1, foo1, foo2)); + } + + private DatafeedConfig createDatafeedConfig(String id, String jobId) { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder(id, jobId); + builder.setIndices(Collections.singletonList("beats*")); + + Map headers = new HashMap<>(); + // Only security headers are updated, grab the first one + String securityHeader = ClientHelper.SECURITY_HEADER_FILTERS.iterator().next(); + headers.put(securityHeader, "SECURITY_"); + builder.setHeaders(headers); + return builder.build(); + } + + private DatafeedConfig putDatafeedConfig(DatafeedConfig config) throws Exception { + this.blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener)); + return config; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index fb82b1c74d0eb..d85d8e1d8cbcd 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -7,7 +7,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; @@ -32,9 +31,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -315,34 +312,6 @@ private Job.Builder createJob(String jobId, List groups) { return builder; } - private void blockingCall(Consumer> function, AtomicReference response, - AtomicReference error) throws InterruptedException { - CountDownLatch latch = new CountDownLatch(1); - ActionListener listener = ActionListener.wrap( - r -> { - response.set(r); - latch.countDown(); - }, - e -> { - error.set(e); - latch.countDown(); - } - ); - - function.accept(listener); - latch.await(); - } - - private T blockingCall(Consumer> function) throws Exception { - AtomicReference exceptionHolder = new AtomicReference<>(); - AtomicReference responseHolder = new AtomicReference<>(); - blockingCall(function, responseHolder, exceptionHolder); - if (exceptionHolder.get() != null) { - assertNotNull(exceptionHolder.get().getMessage(), exceptionHolder.get()); - } - return responseHolder.get(); - } - private Job putJob(Job.Builder job) throws Exception { Job builtJob = job.build(new Date()); this.blockingCall(actionListener -> jobConfigProvider.putJob(builtJob, actionListener)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index 09651f554d848..1bf43a0dc72c0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; @@ -500,7 +501,7 @@ private void indexScheduledEvents(List events) throws IOExceptio for (ScheduledEvent event : events) { IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")); indexRequest.source(event.toXContent(builder, params)); bulkRequest.add(indexRequest); } @@ -543,7 +544,7 @@ private void indexFilters(List filters) throws IOException { for (MlFilter filter : filters) { IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")); indexRequest.source(filter.toXContent(builder, params)); bulkRequest.add(indexRequest); } @@ -573,7 +574,7 @@ private void indexCalendars(List calendars) throws IOException { for (Calendar calendar: calendars) { IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true")); indexRequest.source(calendar.toXContent(builder, params)); bulkRequest.add(indexRequest); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcherTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcherTests.java new file mode 100644 index 0000000000000..4a9a696866e43 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ExpandedIdsMatcherTests.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isOneOf; + +public class ExpandedIdsMatcherTests extends ESTestCase { + + public void testMatchingJobIds() { + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(new String[] {"*"}, false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertTrue(requiredMatches.hasUnmatchedIds()); + requiredMatches.filterMatchedIds(Collections.singletonList("foo")); + assertFalse(requiredMatches.hasUnmatchedIds()); + assertThat(requiredMatches.unmatchedIds(), empty()); + + requiredMatches = new ExpandedIdsMatcher(ExpandedIdsMatcher.tokenizeExpression(""), false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Collections.singletonList("foo")); + assertThat(requiredMatches.unmatchedIds(), empty()); + + requiredMatches = new ExpandedIdsMatcher(ExpandedIdsMatcher.tokenizeExpression(null), false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Collections.singletonList("foo")); + assertThat(requiredMatches.unmatchedIds(), empty()); + + requiredMatches = new ExpandedIdsMatcher(ExpandedIdsMatcher.tokenizeExpression(null), false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Collections.emptyList()); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertThat(requiredMatches.unmatchedIds().get(0), equalTo("*")); + + requiredMatches = new ExpandedIdsMatcher(ExpandedIdsMatcher.tokenizeExpression("_all"), false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Collections.singletonList("foo")); + assertThat(requiredMatches.unmatchedIds(), empty()); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*"}, false); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Arrays.asList("foo1","foo2")); + assertThat(requiredMatches.unmatchedIds(), empty()); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches.unmatchedIds(), hasSize(2)); + requiredMatches.filterMatchedIds(Arrays.asList("foo1","foo2")); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertEquals("bar", requiredMatches.unmatchedIds().get(0)); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches.unmatchedIds(), hasSize(2)); + requiredMatches.filterMatchedIds(Arrays.asList("foo1","bar")); + assertFalse(requiredMatches.hasUnmatchedIds()); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*","bar"}, false); + assertThat(requiredMatches.unmatchedIds(), hasSize(2)); + requiredMatches.filterMatchedIds(Collections.singletonList("bar")); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertEquals("foo*", requiredMatches.unmatchedIds().get(0)); + + requiredMatches = new ExpandedIdsMatcher(ExpandedIdsMatcher.tokenizeExpression("foo,bar,baz,wild*"), false); + assertThat(requiredMatches.unmatchedIds(), hasSize(4)); + requiredMatches.filterMatchedIds(Arrays.asList("foo","baz")); + assertThat(requiredMatches.unmatchedIds(), hasSize(2)); + assertThat(requiredMatches.unmatchedIds().get(0), isOneOf("bar", "wild*")); + assertThat(requiredMatches.unmatchedIds().get(1), isOneOf("bar", "wild*")); + } + + public void testMatchingJobIds_allowNoJobs() { + // wildcard all with allow no jobs + ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(new String[] {"*"}, true); + assertThat(requiredMatches.unmatchedIds(), empty()); + assertFalse(requiredMatches.hasUnmatchedIds()); + requiredMatches.filterMatchedIds(Collections.emptyList()); + assertThat(requiredMatches.unmatchedIds(), empty()); + assertFalse(requiredMatches.hasUnmatchedIds()); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*","bar"}, true); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertTrue(requiredMatches.hasUnmatchedIds()); + requiredMatches.filterMatchedIds(Collections.singletonList("bar")); + assertThat(requiredMatches.unmatchedIds(), empty()); + assertFalse(requiredMatches.hasUnmatchedIds()); + + requiredMatches = new ExpandedIdsMatcher(new String[] {"foo*","bar"}, true); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + requiredMatches.filterMatchedIds(Collections.emptyList()); + assertThat(requiredMatches.unmatchedIds(), hasSize(1)); + assertEquals("bar", requiredMatches.unmatchedIds().get(0)); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java deleted file mode 100644 index 04bcd57e64fc0..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProviderTests.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.persistence; - -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedList; - -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.isOneOf; - -public class JobConfigProviderTests extends ESTestCase { - - public void testMatchingJobIds() { - LinkedList requiredMatches = JobConfigProvider.requiredMatches(new String[] {"*"}, false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(""), false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(null), false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression(null), false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); - assertThat(requiredMatches, hasSize(1)); - assertThat(requiredMatches.get(0).getId(), equalTo("*")); - - requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression("_all"), false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("foo")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*"}, false); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","foo2")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); - assertThat(requiredMatches, hasSize(2)); - JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","foo2")); - assertThat(requiredMatches, hasSize(1)); - assertEquals("bar", requiredMatches.get(0).getId()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); - assertThat(requiredMatches, hasSize(2)); - JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo1","bar")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, false); - assertThat(requiredMatches, hasSize(2)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("bar")); - assertThat(requiredMatches, hasSize(1)); - assertEquals("foo*", requiredMatches.get(0).getId()); - - requiredMatches = JobConfigProvider.requiredMatches(JobConfigProvider.tokenizeExpression("foo,bar,baz,wild*"), false); - assertThat(requiredMatches, hasSize(4)); - JobConfigProvider.filterMatchedIds(requiredMatches, Arrays.asList("foo","baz")); - assertThat(requiredMatches, hasSize(2)); - assertThat(requiredMatches.get(0).getId(), isOneOf("bar", "wild*")); - assertThat(requiredMatches.get(1).getId(), isOneOf("bar", "wild*")); - } - - public void testMatchingJobIds_allowNoJobs() { - // wildcard all with allow no jobs - LinkedList requiredMatches = JobConfigProvider.requiredMatches(new String[] {"*"}, true); - assertThat(requiredMatches, empty()); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, true); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.singletonList("bar")); - assertThat(requiredMatches, empty()); - - requiredMatches = JobConfigProvider.requiredMatches(new String[] {"foo*","bar"}, true); - assertThat(requiredMatches, hasSize(1)); - JobConfigProvider.filterMatchedIds(requiredMatches, Collections.emptyList()); - assertThat(requiredMatches, hasSize(1)); - assertEquals("bar", requiredMatches.get(0).getId()); - } -} From 7b569991e9e2119005ba9ce1db04f0aba3bee8ed Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 29 Aug 2018 17:04:59 +0100 Subject: [PATCH 04/57] [ML] Change JobManager to work with Job config in index (#33064) --- .../elasticsearch/xpack/core/ml/MlTasks.java | 24 +- .../persistence/ElasticsearchMappings.java | 3 - .../ml/job/results/ReservedFieldNames.java | 1 - .../xpack/ml/MachineLearning.java | 2 +- .../action/TransportDeleteCalendarAction.java | 7 +- .../TransportDeleteCalendarEventAction.java | 6 +- .../TransportDeleteModelSnapshotAction.java | 64 +-- .../ml/action/TransportForecastJobAction.java | 94 ++-- .../ml/action/TransportGetBucketsAction.java | 47 +- .../action/TransportGetCategoriesAction.java | 15 +- .../action/TransportGetInfluencersAction.java | 29 +- .../ml/action/TransportGetJobsAction.java | 14 +- .../TransportGetModelSnapshotsAction.java | 18 +- .../TransportGetOverallBucketsAction.java | 35 +- .../ml/action/TransportGetRecordsAction.java | 29 +- .../ml/action/TransportJobTaskAction.java | 4 - .../TransportPostCalendarEventsAction.java | 6 +- .../TransportRevertModelSnapshotAction.java | 33 +- .../TransportUpdateCalendarJobAction.java | 6 +- .../action/TransportUpdateFilterAction.java | 6 +- .../xpack/ml/job/JobManager.java | 412 +++++++++--------- .../ml/job/persistence/JobConfigProvider.java | 254 +++++++++-- .../autodetect/AutodetectProcessManager.java | 129 +++--- .../MachineLearningLicensingTests.java | 8 + .../xpack/ml/MlSingleNodeTestCase.java | 2 +- .../elasticsearch/xpack/ml/MlTasksTests.java | 18 + .../ml/integration/JobConfigProviderIT.java | 135 +++++- .../xpack/ml/job/JobManagerTests.java | 295 +++++++++---- .../ml/job/persistence/MockClientBuilder.java | 58 +++ .../AutodetectProcessManagerTests.java | 95 ++-- 30 files changed, 1256 insertions(+), 593 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index 5c17271738e32..f421ba7bf4ad8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -12,8 +12,15 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; + public final class MlTasks { + public static final String JOB_TASK_PREFIX = "job-"; + public static final String DATAFEED_TASK_PREFIX = "datafeed-"; + private MlTasks() { } @@ -22,7 +29,7 @@ private MlTasks() { * A datafeed id can be used as a job id, because they are stored separately in cluster state. */ public static String jobTaskId(String jobId) { - return "job-" + jobId; + return JOB_TASK_PREFIX + jobId; } /** @@ -30,7 +37,7 @@ public static String jobTaskId(String jobId) { * A job id can be used as a datafeed id, because they are stored separately in cluster state. */ public static String datafeedTaskId(String datafeedId) { - return "datafeed-" + datafeedId; + return DATAFEED_TASK_PREFIX + datafeedId; } @Nullable @@ -67,4 +74,17 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis return DatafeedState.STOPPED; } } + + /** + * The job Ids of anomaly detector job tasks + * @param tasks Active tasks + * @return The job Ids of anomaly detector job tasks + */ + public static Set openJobIds(PersistentTasksCustomMetaData tasks) { + Collection> activeTasks = tasks.tasks(); + + return activeTasks.stream().filter(t -> t.getId().startsWith(JOB_TASK_PREFIX)) + .map(t -> t.getId().substring(JOB_TASK_PREFIX.length())) + .collect(Collectors.toSet()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 3d560949963d6..55dc288b7cfa4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -248,9 +248,6 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio .startObject(AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName()) .field(TYPE, BOOLEAN) .endObject() - .startObject(AnalysisConfig.USE_PER_PARTITION_NORMALIZATION.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() .endObject() .endObject() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index fd1c6a9d02fb9..baff9f3a2d51d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -210,7 +210,6 @@ public final class ReservedFieldNames { AnalysisConfig.OVERLAPPING_BUCKETS.getPreferredName(), AnalysisConfig.RESULT_FINALIZATION_WINDOW.getPreferredName(), AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName(), - AnalysisConfig.USE_PER_PARTITION_NORMALIZATION.getPreferredName(), AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(), AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 800995bb25bdf..0e22c20e43416 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -366,7 +366,7 @@ public Collection createComponents(Client client, ClusterService cluster Auditor auditor = new Auditor(client, clusterService.getNodeName()); JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings); UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(settings, client, clusterService, threadPool); - JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, client, notifier); + JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, threadPool, client, notifier); JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(settings, client); JobResultsPersister jobResultsPersister = new JobResultsPersister(settings, client); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java index 4c923f2f77ce3..8850439b24a5a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java @@ -62,8 +62,11 @@ protected void doExecute(Task task, DeleteCalendarAction.Request request, Action listener.onFailure(new ResourceNotFoundException("No calendar with id [" + calendarId + "]")); return; } - jobManager.updateProcessOnCalendarChanged(calendar.getJobIds()); - listener.onResponse(new AcknowledgedResponse(true)); + + jobManager.updateProcessOnCalendarChanged(calendar.getJobIds(), ActionListener.wrap( + r -> listener.onResponse(new AcknowledgedResponse(true)), + listener::onFailure + )); }, listener::onFailure)); }, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java index 854ff19e16f41..b17cf6d1a2b7d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java @@ -102,8 +102,10 @@ public void onResponse(DeleteResponse response) { if (response.status() == RestStatus.NOT_FOUND) { listener.onFailure(new ResourceNotFoundException("No event with id [" + eventId + "]")); } else { - jobManager.updateProcessOnCalendarChanged(calendar.getJobIds()); - listener.onResponse(new AcknowledgedResponse(true)); + jobManager.updateProcessOnCalendarChanged(calendar.getJobIds(), ActionListener.wrap( + r -> listener.onResponse(new AcknowledgedResponse(true)), + listener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java index 6d0721b03d972..f0edf48f7eb34 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java @@ -12,17 +12,15 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; @@ -33,19 +31,19 @@ public class TransportDeleteModelSnapshotAction extends HandledTransportAction { private final Client client; + private final JobManager jobManager; private final JobResultsProvider jobResultsProvider; - private final ClusterService clusterService; private final Auditor auditor; @Inject public TransportDeleteModelSnapshotAction(Settings settings, TransportService transportService, ActionFilters actionFilters, - JobResultsProvider jobResultsProvider, ClusterService clusterService, Client client, + JobResultsProvider jobResultsProvider, Client client, JobManager jobManager, Auditor auditor) { super(settings, DeleteModelSnapshotAction.NAME, transportService, actionFilters, DeleteModelSnapshotAction.Request::new); this.client = client; + this.jobManager = jobManager; this.jobResultsProvider = jobResultsProvider; - this.clusterService = clusterService; this.auditor = auditor; } @@ -70,32 +68,40 @@ protected void doExecute(Task task, DeleteModelSnapshotAction.Request request, ModelSnapshot deleteCandidate = deleteCandidates.get(0); // Verify the snapshot is not being used - Job job = JobManager.getJobOrThrowIfUnknown(request.getJobId(), clusterService.state()); - String currentModelInUse = job.getModelSnapshotId(); - if (currentModelInUse != null && currentModelInUse.equals(request.getSnapshotId())) { - throw new IllegalArgumentException(Messages.getMessage(Messages.REST_CANNOT_DELETE_HIGHEST_PRIORITY, - request.getSnapshotId(), request.getJobId())); - } + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + String currentModelInUse = job.getModelSnapshotId(); + if (currentModelInUse != null && currentModelInUse.equals(request.getSnapshotId())) { + listener.onFailure( + new IllegalArgumentException(Messages.getMessage(Messages.REST_CANNOT_DELETE_HIGHEST_PRIORITY, + request.getSnapshotId(), request.getJobId()))); + return; + } + + // Delete the snapshot and any associated state files + JobDataDeleter deleter = new JobDataDeleter(client, request.getJobId()); + deleter.deleteModelSnapshots(Collections.singletonList(deleteCandidate), + new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + String msg = Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, + deleteCandidate.getSnapshotId(), deleteCandidate.getDescription()); - // Delete the snapshot and any associated state files - JobDataDeleter deleter = new JobDataDeleter(client, request.getJobId()); - deleter.deleteModelSnapshots(Collections.singletonList(deleteCandidate), new ActionListener() { - @Override - public void onResponse(BulkResponse bulkResponse) { - String msg = Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, deleteCandidate.getSnapshotId(), - deleteCandidate.getDescription()); - auditor.info(request.getJobId(), msg); - logger.debug("[{}] {}", request.getJobId(), msg); - // We don't care about the bulk response, just that it succeeded - listener.onResponse(new AcknowledgedResponse(true)); - } + auditor.info(request.getJobId(), msg); + logger.debug("[{}] {}", request.getJobId(), msg); + // We don't care about the bulk response, just that it succeeded + listener.onResponse(new AcknowledgedResponse(true)); + } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + }, + listener::onFailure + )); }, listener::onFailure); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java index ad9e6a7c2630a..c418bcb9b4721 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; @@ -41,14 +40,17 @@ public class TransportForecastJobAction extends TransportJobTaskAction listener) { - ClusterState state = clusterService.state(); - Job job = JobManager.getJobOrThrowIfUnknown(task.getJobId(), state); - validate(job, request); + jobManager.getJob(task.getJobId(), ActionListener.wrap( + job -> { + validate(job, request); - ForecastParams.Builder paramsBuilder = ForecastParams.builder(); + ForecastParams.Builder paramsBuilder = ForecastParams.builder(); - if (request.getDuration() != null) { - paramsBuilder.duration(request.getDuration()); - } + if (request.getDuration() != null) { + paramsBuilder.duration(request.getDuration()); + } - if (request.getExpiresIn() != null) { - paramsBuilder.expiresIn(request.getExpiresIn()); - } + if (request.getExpiresIn() != null) { + paramsBuilder.expiresIn(request.getExpiresIn()); + } - // tmp storage might be null, we do not log here, because it might not be - // required - Path tmpStorage = processManager.tryGetTmpStorage(task, FORECAST_LOCAL_STORAGE_LIMIT); - if (tmpStorage != null) { - paramsBuilder.tmpStorage(tmpStorage.toString()); - } + // tmp storage might be null, we do not log here, because it might not be + // required + Path tmpStorage = processManager.tryGetTmpStorage(task, FORECAST_LOCAL_STORAGE_LIMIT); + if (tmpStorage != null) { + paramsBuilder.tmpStorage(tmpStorage.toString()); + } - ForecastParams params = paramsBuilder.build(); - processManager.forecastJob(task, params, e -> { - if (e == null) { - Consumer forecastRequestStatsHandler = forecastRequestStats -> { - if (forecastRequestStats == null) { - // paranoia case, it should not happen that we do not retrieve a result - listener.onFailure(new ElasticsearchException( - "Cannot run forecast: internal error, please check the logs")); - } else if (forecastRequestStats.getStatus() == ForecastRequestStats.ForecastRequestStatus.FAILED) { - List messages = forecastRequestStats.getMessages(); - if (messages.size() > 0) { - listener.onFailure(ExceptionsHelper.badRequestException("Cannot run forecast: " - + messages.get(0))); + ForecastParams params = paramsBuilder.build(); + processManager.forecastJob(task, params, e -> { + if (e == null) { +; getForecastRequestStats(request.getJobId(), params.getForecastId(), listener); } else { - // paranoia case, it should not be possible to have an empty message list - listener.onFailure( - new ElasticsearchException( - "Cannot run forecast: internal error, please check the logs")); + listener.onFailure(e); } - } else { - listener.onResponse(new ForecastJobAction.Response(true, params.getForecastId())); - } - }; + }); + }, + listener::onFailure + )); + } - jobResultsProvider.getForecastRequestStats(request.getJobId(), params.getForecastId(), - forecastRequestStatsHandler, listener::onFailure); + private void getForecastRequestStats(String jobId, String forecastId, ActionListener listener) { + Consumer forecastRequestStatsHandler = forecastRequestStats -> { + if (forecastRequestStats == null) { + // paranoia case, it should not happen that we do not retrieve a result + listener.onFailure(new ElasticsearchException( + "Cannot run forecast: internal error, please check the logs")); + } else if (forecastRequestStats.getStatus() == ForecastRequestStats.ForecastRequestStatus.FAILED) { + List messages = forecastRequestStats.getMessages(); + if (messages.size() > 0) { + listener.onFailure(ExceptionsHelper.badRequestException("Cannot run forecast: " + + messages.get(0))); + } else { + // paranoia case, it should not be possible to have an empty message list + listener.onFailure( + new ElasticsearchException( + "Cannot run forecast: internal error, please check the logs")); + } } else { - listener.onFailure(e); + listener.onResponse(new ForecastJobAction.Response(true, forecastId)); } - }); + }; + + jobResultsProvider.getForecastRequestStats(jobId, forecastId, forecastRequestStatsHandler, listener::onFailure); } static void validate(Job job, ForecastJobAction.Request request) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java index 377c5a6239289..5e0bafac71475 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java @@ -39,28 +39,33 @@ public TransportGetBucketsAction(Settings settings, TransportService transportSe @Override protected void doExecute(Task task, GetBucketsAction.Request request, ActionListener listener) { - jobManager.getJobOrThrowIfUnknown(request.getJobId()); + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + BucketsQueryBuilder query = + new BucketsQueryBuilder().expand(request.isExpand()) + .includeInterim(request.isExcludeInterim() == false) + .start(request.getStart()) + .end(request.getEnd()) + .anomalyScoreThreshold(request.getAnomalyScore()) + .sortField(request.getSort()) + .sortDescending(request.isDescending()); - BucketsQueryBuilder query = - new BucketsQueryBuilder().expand(request.isExpand()) - .includeInterim(request.isExcludeInterim() == false) - .start(request.getStart()) - .end(request.getEnd()) - .anomalyScoreThreshold(request.getAnomalyScore()) - .sortField(request.getSort()) - .sortDescending(request.isDescending()); + if (request.getPageParams() != null) { + query.from(request.getPageParams().getFrom()) + .size(request.getPageParams().getSize()); + } + if (request.getTimestamp() != null) { + query.timestamp(request.getTimestamp()); + } else { + query.start(request.getStart()); + query.end(request.getEnd()); + } + jobResultsProvider.buckets(request.getJobId(), query, q -> + listener.onResponse(new GetBucketsAction.Response(q)), listener::onFailure, client); - if (request.getPageParams() != null) { - query.from(request.getPageParams().getFrom()) - .size(request.getPageParams().getSize()); - } - if (request.getTimestamp() != null) { - query.timestamp(request.getTimestamp()); - } else { - query.start(request.getStart()); - query.end(request.getEnd()); - } - jobResultsProvider.buckets(request.getJobId(), query, q -> - listener.onResponse(new GetBucketsAction.Response(q)), listener::onFailure, client); + }, + listener::onFailure + + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java index 4bac6321a3ebf..d90cd59ed0215 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java @@ -38,11 +38,14 @@ public TransportGetCategoriesAction(Settings settings, TransportService transpor @Override protected void doExecute(Task task, GetCategoriesAction.Request request, ActionListener listener) { - jobManager.getJobOrThrowIfUnknown(request.getJobId()); - - Integer from = request.getPageParams() != null ? request.getPageParams().getFrom() : null; - Integer size = request.getPageParams() != null ? request.getPageParams().getSize() : null; - jobResultsProvider.categoryDefinitions(request.getJobId(), request.getCategoryId(), true, from, size, - r -> listener.onResponse(new GetCategoriesAction.Response(r)), listener::onFailure, client); + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + Integer from = request.getPageParams() != null ? request.getPageParams().getFrom() : null; + Integer size = request.getPageParams() != null ? request.getPageParams().getSize() : null; + jobResultsProvider.categoryDefinitions(request.getJobId(), request.getCategoryId(), true, from, size, + r -> listener.onResponse(new GetCategoriesAction.Response(r)), listener::onFailure, client); + }, + listener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java index e62538831b5cb..9967262ba1070 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java @@ -39,18 +39,21 @@ public TransportGetInfluencersAction(Settings settings, TransportService transpo @Override protected void doExecute(Task task, GetInfluencersAction.Request request, ActionListener listener) { - jobManager.getJobOrThrowIfUnknown(request.getJobId()); - - InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder() - .includeInterim(request.isExcludeInterim() == false) - .start(request.getStart()) - .end(request.getEnd()) - .from(request.getPageParams().getFrom()) - .size(request.getPageParams().getSize()) - .influencerScoreThreshold(request.getInfluencerScore()) - .sortField(request.getSort()) - .sortDescending(request.isDescending()).build(); - jobResultsProvider.influencers(request.getJobId(), query, - page -> listener.onResponse(new GetInfluencersAction.Response(page)), listener::onFailure, client); + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder() + .includeInterim(request.isExcludeInterim() == false) + .start(request.getStart()) + .end(request.getEnd()) + .from(request.getPageParams().getFrom()) + .size(request.getPageParams().getSize()) + .influencerScoreThreshold(request.getInfluencerScore()) + .sortField(request.getSort()) + .sortDescending(request.isDescending()).build(); + jobResultsProvider.influencers(request.getJobId(), query, + page -> listener.onResponse(new GetInfluencersAction.Response(page)), listener::onFailure, client); + }, + listener::onFailure) + ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java index 0932d19723237..375da65e77edb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java @@ -18,9 +18,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.GetJobsAction; -import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.ml.job.JobManager; -import org.elasticsearch.xpack.core.ml.job.config.Job; public class TransportGetJobsAction extends TransportMasterNodeReadAction { @@ -47,11 +45,15 @@ protected GetJobsAction.Response newResponse() { } @Override - protected void masterOperation(GetJobsAction.Request request, ClusterState state, ActionListener listener) - throws Exception { + protected void masterOperation(GetJobsAction.Request request, ClusterState state, + ActionListener listener) { logger.debug("Get job '{}'", request.getJobId()); - QueryPage jobs = jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), state); - listener.onResponse(new GetJobsAction.Response(jobs)); + jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + jobs -> { + listener.onResponse(new GetJobsAction.Response(jobs)); + }, + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java index 7a0e0b1c4deb1..a690d6ef8593c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java @@ -43,13 +43,17 @@ protected void doExecute(Task task, GetModelSnapshotsAction.Request request, request.getJobId(), request.getSnapshotId(), request.getPageParams().getFrom(), request.getPageParams().getSize(), request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder()); - jobManager.getJobOrThrowIfUnknown(request.getJobId()); - - jobResultsProvider.modelSnapshots(request.getJobId(), request.getPageParams().getFrom(), request.getPageParams().getSize(), - request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), request.getSnapshotId(), - page -> { - listener.onResponse(new GetModelSnapshotsAction.Response(clearQuantiles(page))); - }, listener::onFailure); + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + jobResultsProvider.modelSnapshots(request.getJobId(), request.getPageParams().getFrom(), + request.getPageParams().getSize(), request.getStart(), request.getEnd(), request.getSort(), + request.getDescOrder(), request.getSnapshotId(), + page -> { + listener.onResponse(new GetModelSnapshotsAction.Response(clearQuantiles(page))); + }, listener::onFailure); + }, + listener::onFailure + )); } public static QueryPage clearQuantiles(QueryPage page) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java index 1ff5bdecf6926..3364f8e08b85b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java @@ -75,23 +75,26 @@ public TransportGetOverallBucketsAction(Settings settings, ThreadPool threadPool } @Override - protected void doExecute(Task task, GetOverallBucketsAction.Request request, - ActionListener listener) { - QueryPage jobsPage = jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), clusterService.state()); - if (jobsPage.count() == 0) { - listener.onResponse(new GetOverallBucketsAction.Response()); - return; - } + protected void doExecute(Task task, GetOverallBucketsAction.Request request, ActionListener listener) { + jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + jobPage -> { + if (jobPage.count() == 0) { + listener.onResponse(new GetOverallBucketsAction.Response()); + return; + } - // As computing and potentially aggregating overall buckets might take a while, - // we run in a different thread to avoid blocking the network thread. - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { - try { - getOverallBuckets(request, jobsPage.results(), listener); - } catch (Exception e) { - listener.onFailure(e); - } - }); + // As computing and potentially aggregating overall buckets might take a while, + // we run in a different thread to avoid blocking the network thread. + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { + try { + getOverallBuckets(request, jobPage.results(), listener); + } catch (Exception e) { + listener.onFailure(e); + } + }); + }, + listener::onFailure + )); } private void getOverallBuckets(GetOverallBucketsAction.Request request, List jobs, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java index 15a78efd9fda2..6c43985af5561 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java @@ -40,18 +40,21 @@ public TransportGetRecordsAction(Settings settings, TransportService transportSe @Override protected void doExecute(Task task, GetRecordsAction.Request request, ActionListener listener) { - jobManager.getJobOrThrowIfUnknown(request.getJobId()); - - RecordsQueryBuilder query = new RecordsQueryBuilder() - .includeInterim(request.isExcludeInterim() == false) - .epochStart(request.getStart()) - .epochEnd(request.getEnd()) - .from(request.getPageParams().getFrom()) - .size(request.getPageParams().getSize()) - .recordScore(request.getRecordScoreFilter()) - .sortField(request.getSort()) - .sortDescending(request.isDescending()); - jobResultsProvider.records(request.getJobId(), query, page -> - listener.onResponse(new GetRecordsAction.Response(page)), listener::onFailure, client); + jobManager.getJob(request.getJobId(), ActionListener.wrap( + job -> { + RecordsQueryBuilder query = new RecordsQueryBuilder() + .includeInterim(request.isExcludeInterim() == false) + .epochStart(request.getStart()) + .epochEnd(request.getEnd()) + .from(request.getPageParams().getFrom()) + .size(request.getPageParams().getSize()) + .recordScore(request.getRecordScoreFilter()) + .sortField(request.getSort()) + .sortDescending(request.isDescending()); + jobResultsProvider.records(request.getJobId(), query, page -> + listener.onResponse(new GetRecordsAction.Response(page)), listener::onFailure, client); + }, + listener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java index 07bb6152e8c1d..bd337e6fd75bc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.tasks.BaseTasksResponse; import org.elasticsearch.action.support.tasks.TransportTasksAction; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; @@ -21,7 +20,6 @@ import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.JobTaskRequest; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import java.util.List; @@ -52,8 +50,6 @@ protected void doExecute(Task task, Request request, ActionListener li String jobId = request.getJobId(); // We need to check whether there is at least an assigned task here, otherwise we cannot redirect to the // node running the job task. - ClusterState state = clusterService.state(); - JobManager.getJobOrThrowIfUnknown(jobId, state); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData.PersistentTask jobTask = MlTasks.getJobTask(jobId, tasks); if (jobTask == null || jobTask.isAssigned() == false) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java index 24bc4ad016e7b..5a1f2eab1fe9d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java @@ -82,8 +82,10 @@ protected void doExecute(Task task, PostCalendarEventsAction.Request request, new ActionListener() { @Override public void onResponse(BulkResponse response) { - jobManager.updateProcessOnCalendarChanged(calendar.getJobIds()); - listener.onResponse(new PostCalendarEventsAction.Response(events)); + jobManager.updateProcessOnCalendarChanged(calendar.getJobIds(), ActionListener.wrap( + r -> listener.onResponse(new PostCalendarEventsAction.Response(events)), + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index 6d5b8bdb0db03..c5096c966a405 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; @@ -72,22 +71,26 @@ protected void masterOperation(RevertModelSnapshotAction.Request request, Cluste logger.debug("Received request to revert to snapshot id '{}' for job '{}', deleting intervening results: {}", request.getSnapshotId(), request.getJobId(), request.getDeleteInterveningResults()); - Job job = JobManager.getJobOrThrowIfUnknown(request.getJobId(), state); - PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - JobState jobState = MlTasks.getJobState(job.getId(), tasks); + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + exists -> { + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + JobState jobState = MlTasks.getJobState(request.getJobId(), tasks); - if (jobState.equals(JobState.CLOSED) == false) { - throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT)); - } + if (jobState.equals(JobState.CLOSED) == false) { + throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT)); + } - getModelSnapshot(request, jobResultsProvider, modelSnapshot -> { - ActionListener wrappedListener = listener; - if (request.getDeleteInterveningResults()) { - wrappedListener = wrapDeleteOldDataListener(wrappedListener, modelSnapshot, request.getJobId()); - wrappedListener = wrapRevertDataCountsListener(wrappedListener, modelSnapshot, request.getJobId()); - } - jobManager.revertSnapshot(request, wrappedListener, modelSnapshot); - }, listener::onFailure); + getModelSnapshot(request, jobResultsProvider, modelSnapshot -> { + ActionListener wrappedListener = listener; + if (request.getDeleteInterveningResults()) { + wrappedListener = wrapDeleteOldDataListener(wrappedListener, modelSnapshot, request.getJobId()); + wrappedListener = wrapRevertDataCountsListener(wrappedListener, modelSnapshot, request.getJobId()); + } + jobManager.revertSnapshot(request, wrappedListener, modelSnapshot); + }, listener::onFailure); + }, + listener::onFailure + )); } private void getModelSnapshot(RevertModelSnapshotAction.Request request, JobResultsProvider provider, Consumer handler, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java index adb3d35765c03..5c50ff34e7b4c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java @@ -40,8 +40,10 @@ protected void doExecute(Task task, UpdateCalendarJobAction.Request request, Act jobResultsProvider.updateCalendar(request.getCalendarId(), jobIdsToAdd, jobIdsToRemove, c -> { - jobManager.updateProcessOnCalendarChanged(c.getJobIds()); - listener.onResponse(new PutCalendarAction.Response(c)); + jobManager.updateProcessOnCalendarChanged(c.getJobIds(), ActionListener.wrap( + r -> listener.onResponse(new PutCalendarAction.Response(c)), + listener::onFailure + )); }, listener::onFailure); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java index abbefa1e4936f..822c4378cc950 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java @@ -115,8 +115,10 @@ private void indexUpdatedFilter(MlFilter filter, long version, UpdateFilterActio executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, new ActionListener() { @Override public void onResponse(IndexResponse indexResponse) { - jobManager.notifyFilterChanged(filter, request.getAddItems(), request.getRemoveItems()); - listener.onResponse(new PutFilterAction.Response(filter)); + jobManager.notifyFilterChanged(filter, request.getAddItems(), request.getRemoveItems(), ActionListener.wrap( + response -> listener.onResponse(new PutFilterAction.Response(filter)), + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 1e97e98c42c3b..170de42a78102 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -11,9 +11,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; @@ -29,7 +27,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -50,6 +48,7 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams; @@ -58,24 +57,25 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * Allows interactions with jobs. The managed interactions include: *
      *
    • creation
      • + *
    • reading
      • *
    • deletion
      • *
    • updating
      • - *
    • starting/stopping of datafeed jobs
      • *
    */ public class JobManager extends AbstractComponent { @@ -88,7 +88,9 @@ public class JobManager extends AbstractComponent { private final ClusterService clusterService; private final Auditor auditor; private final Client client; + private final ThreadPool threadPool; private final UpdateJobProcessNotifier updateJobProcessNotifier; + private final JobConfigProvider jobConfigProvider; private volatile ByteSizeValue maxModelMemoryLimit; @@ -96,7 +98,7 @@ public class JobManager extends AbstractComponent { * Create a JobManager */ public JobManager(Environment environment, Settings settings, JobResultsProvider jobResultsProvider, - ClusterService clusterService, Auditor auditor, + ClusterService clusterService, Auditor auditor, ThreadPool threadPool, Client client, UpdateJobProcessNotifier updateJobProcessNotifier) { super(settings); this.environment = environment; @@ -104,7 +106,9 @@ public JobManager(Environment environment, Settings settings, JobResultsProvider this.clusterService = Objects.requireNonNull(clusterService); this.auditor = Objects.requireNonNull(auditor); this.client = Objects.requireNonNull(client); + this.threadPool = Objects.requireNonNull(threadPool); this.updateJobProcessNotifier = updateJobProcessNotifier; + this.jobConfigProvider = new JobConfigProvider(client, settings); maxModelMemoryLimit = MachineLearningField.MAX_MODEL_MEMORY_LIMIT.get(settings); clusterService.getClusterSettings() @@ -115,35 +119,46 @@ private void setMaxModelMemoryLimit(ByteSizeValue maxModelMemoryLimit) { this.maxModelMemoryLimit = maxModelMemoryLimit; } + public void jobExists(String jobId, ActionListener listener) { + jobConfigProvider.checkJobExists(jobId, listener); + } + /** * Gets the job that matches the given {@code jobId}. * * @param jobId the jobId - * @return The {@link Job} matching the given {code jobId} - * @throws ResourceNotFoundException if no job matches {@code jobId} + * @param jobListener the Job listener. If no job matches {@code jobId} + * a ResourceNotFoundException is returned */ - public Job getJobOrThrowIfUnknown(String jobId) { - return getJobOrThrowIfUnknown(jobId, clusterService.state()); + public void getJob(String jobId, ActionListener jobListener) { + jobConfigProvider.getJob(jobId, ActionListener.wrap( + r -> jobListener.onResponse(r.build()), // TODO JIndex we shouldn't be building the job here + e -> { + if (e instanceof ResourceNotFoundException) { + // Try to get the job from the cluster state + getJobFromClusterState(jobId, jobListener); + } else { + jobListener.onFailure(e); + } + } + )); } /** - * Gets the job that matches the given {@code jobId}. + * Read a job from the cluster state. + * The job is returned on the same thread even though a listener is used. * * @param jobId the jobId - * @param clusterState the cluster state - * @return The {@link Job} matching the given {code jobId} - * @throws ResourceNotFoundException if no job matches {@code jobId} + * @param jobListener the Job listener. If no job matches {@code jobId} + * a ResourceNotFoundException is returned */ - public static Job getJobOrThrowIfUnknown(String jobId, ClusterState clusterState) { - Job job = MlMetadata.getMlMetadata(clusterState).getJobs().get(jobId); + private void getJobFromClusterState(String jobId, ActionListener jobListener) { + Job job = MlMetadata.getMlMetadata(clusterService.state()).getJobs().get(jobId); if (job == null) { - throw ExceptionsHelper.missingJobException(jobId); + jobListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + jobListener.onResponse(job); } - return job; - } - - private Set expandJobIds(String expression, boolean allowNoJobs, ClusterState clusterState) { - return MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoJobs); } /** @@ -151,19 +166,45 @@ private Set expandJobIds(String expression, boolean allowNoJobs, Cluster * Note that when the {@code jobId} is {@link MetaData#ALL} all jobs are returned. * * @param expression the jobId or an expression matching jobIds - * @param clusterState the cluster state * @param allowNoJobs if {@code false}, an error is thrown when no job matches the {@code jobId} - * @return A {@link QueryPage} containing the matching {@code Job}s + * @param jobsListener The jobs listener */ - public QueryPage expandJobs(String expression, boolean allowNoJobs, ClusterState clusterState) { - Set expandedJobIds = expandJobIds(expression, allowNoJobs, clusterState); + public void expandJobs(String expression, boolean allowNoJobs, ActionListener> jobsListener) { + Map clusterStateJobs = expandJobsFromClusterState(expression, allowNoJobs, clusterService.state()); + + jobConfigProvider.expandJobs(expression, allowNoJobs, ActionListener.wrap( + jobBuilders -> { + // Check for duplicate jobs + for (Job.Builder jb : jobBuilders) { + if (clusterStateJobs.containsKey(jb.getId())) { + jobsListener.onFailure(new IllegalStateException("Job [" + jb.getId() + "] configuration " + + "exists in both clusterstate and index")); + return; + } + } + + // Merge cluster state and index jobs + List jobs = new ArrayList<>(); + for (Job.Builder jb : jobBuilders) { + jobs.add(jb.build()); + } + + jobs.addAll(clusterStateJobs.values()); + Collections.sort(jobs, Comparator.comparing(Job::getId)); + jobsListener.onResponse(new QueryPage<>(jobs, jobs.size(), Job.RESULTS_FIELD)); + }, + jobsListener::onFailure + )); + } + + private Map expandJobsFromClusterState(String expression, boolean allowNoJobs, ClusterState clusterState) { + Set expandedJobIds = MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoJobs); MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - List jobs = new ArrayList<>(); + Map jobIdToJob = new HashMap<>(); for (String expandedJobId : expandedJobIds) { - jobs.add(mlMetadata.getJobs().get(expandedJobId)); + jobIdToJob.put(expandedJobId, mlMetadata.getJobs().get(expandedJobId)); } - logger.debug("Returning jobs matching [" + expression + "]"); - return new QueryPage<>(jobs, jobs.size(), Job.RESULTS_FIELD); + return jobIdToJob; } /** @@ -183,7 +224,7 @@ static void validateCategorizationAnalyzer(Job.Builder jobBuilder, AnalysisRegis } /** - * Stores a job in the cluster state + * Stores the anomaly job configuration */ public void putJob(PutJobAction.Request request, AnalysisRegistry analysisRegistry, ClusterState state, ActionListener actionListener) throws IOException { @@ -197,9 +238,7 @@ public void putJob(PutJobAction.Request request, AnalysisRegistry analysisRegist deprecationLogger.deprecated("Creating jobs with delimited data format is deprecated. Please use xcontent instead."); } - // pre-flight check, not necessarily required, but avoids figuring this out while on the CS update thread - XPackPlugin.checkReadyForXPackCustomMetadata(state); - + // Check for the job in the cluster state first MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(state); if (currentMlMetadata.getJobs().containsKey(job.getId())) { actionListener.onFailure(ExceptionsHelper.jobAlreadyExists(job.getId())); @@ -210,19 +249,13 @@ public void putJob(PutJobAction.Request request, AnalysisRegistry analysisRegist @Override public void onResponse(Boolean indicesCreated) { - clusterService.submitStateUpdateTask("put-job-" + job.getId(), - new AckedClusterStateUpdateTask(request, actionListener) { - @Override - protected PutJobAction.Response newResponse(boolean acknowledged) { - auditor.info(job.getId(), Messages.getMessage(Messages.JOB_AUDIT_CREATED)); - return new PutJobAction.Response(job); - } - - @Override - public ClusterState execute(ClusterState currentState) { - return updateClusterState(job, false, currentState); - } - }); + jobConfigProvider.putJob(job, ActionListener.wrap( + response -> { + auditor.info(job.getId(), Messages.getMessage(Messages.JOB_AUDIT_CREATED)); + actionListener.onResponse(new PutJobAction.Response(job)); + }, + actionListener::onFailure + )); } @Override @@ -252,13 +285,53 @@ public void onFailure(Exception e) { } public void updateJob(UpdateJobAction.Request request, ActionListener actionListener) { - Job job = getJobOrThrowIfUnknown(request.getJobId()); - validate(request.getJobUpdate(), job, ActionListener.wrap( - nullValue -> internalJobUpdate(request, actionListener), - actionListener::onFailure)); + + ActionListener postUpdateAction; + + // Autodetect must be updated if the fields that the C++ uses are changed + if (request.getJobUpdate().isAutodetectProcessUpdate()) { + postUpdateAction = ActionListener.wrap( + updatedJob -> { + JobUpdate jobUpdate = request.getJobUpdate(); + if (isJobOpen(clusterService.state(), request.getJobId())) { + updateJobProcessNotifier.submitJobUpdate(UpdateParams.fromJobUpdate(jobUpdate), ActionListener.wrap( + isUpdated -> { + if (isUpdated) { + auditJobUpdatedIfNotInternal(request); + } + }, e -> { + // No need to do anything + } + )); + } + actionListener.onResponse(new PutJobAction.Response(updatedJob)); + }, + actionListener::onFailure + ); + } else { + postUpdateAction = ActionListener.wrap(job -> { + logger.debug("[{}] No process update required for job update: {}", () -> request.getJobId(), () -> { + try { + XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); + request.getJobUpdate().toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); + return Strings.toString(jsonBuilder); + } catch (IOException e) { + return "(unprintable due to " + e.getMessage() + ")"; + } + }); + + auditJobUpdatedIfNotInternal(request); + actionListener.onResponse(new PutJobAction.Response(job)); + }, + actionListener::onFailure); + } + + + jobConfigProvider.updateJobWithValidation(request.getJobId(), request.getJobUpdate(), maxModelMemoryLimit, + this::validate, postUpdateAction); } - private void validate(JobUpdate jobUpdate, Job job, ActionListener handler) { + private void validate(Job job, JobUpdate jobUpdate, ActionListener handler) { ChainTaskExecutor chainTaskExecutor = new ChainTaskExecutor(client.threadPool().executor( MachineLearning.UTILITY_THREAD_POOL_NAME), true); validateModelSnapshotIdUpdate(job, jobUpdate.getModelSnapshotId(), chainTaskExecutor); @@ -317,86 +390,6 @@ private void validateAnalysisLimitsUpdate(Job job, AnalysisLimits newLimits, Cha }); } - private void internalJobUpdate(UpdateJobAction.Request request, ActionListener actionListener) { - if (request.isWaitForAck()) { - // Use the ack cluster state update - clusterService.submitStateUpdateTask("update-job-" + request.getJobId(), - new AckedClusterStateUpdateTask(request, actionListener) { - private AtomicReference updatedJob = new AtomicReference<>(); - - @Override - protected PutJobAction.Response newResponse(boolean acknowledged) { - return new PutJobAction.Response(updatedJob.get()); - } - - @Override - public ClusterState execute(ClusterState currentState) { - Job job = getJobOrThrowIfUnknown(request.getJobId(), currentState); - updatedJob.set(request.getJobUpdate().mergeWithJob(job, maxModelMemoryLimit)); - return updateClusterState(updatedJob.get(), true, currentState); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - afterClusterStateUpdate(newState, request); - } - }); - } else { - clusterService.submitStateUpdateTask("update-job-" + request.getJobId(), new ClusterStateUpdateTask() { - private AtomicReference updatedJob = new AtomicReference<>(); - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - Job job = getJobOrThrowIfUnknown(request.getJobId(), currentState); - updatedJob.set(request.getJobUpdate().mergeWithJob(job, maxModelMemoryLimit)); - return updateClusterState(updatedJob.get(), true, currentState); - } - - @Override - public void onFailure(String source, Exception e) { - actionListener.onFailure(e); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - afterClusterStateUpdate(newState, request); - actionListener.onResponse(new PutJobAction.Response(updatedJob.get())); - } - }); - } - } - - private void afterClusterStateUpdate(ClusterState newState, UpdateJobAction.Request request) { - JobUpdate jobUpdate = request.getJobUpdate(); - - // Change is required if the fields that the C++ uses are being updated - boolean processUpdateRequired = jobUpdate.isAutodetectProcessUpdate(); - - if (processUpdateRequired && isJobOpen(newState, request.getJobId())) { - updateJobProcessNotifier.submitJobUpdate(UpdateParams.fromJobUpdate(jobUpdate), ActionListener.wrap( - isUpdated -> { - if (isUpdated) { - auditJobUpdatedIfNotInternal(request); - } - }, e -> { - // No need to do anything - } - )); - } else { - logger.debug("[{}] No process update required for job update: {}", () -> request.getJobId(), () -> { - try { - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - jobUpdate.toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); - return Strings.toString(jsonBuilder); - } catch (IOException e) { - return "(unprintable due to " + e.getMessage() + ")"; - } - }); - - auditJobUpdatedIfNotInternal(request); - } - } - private void auditJobUpdatedIfNotInternal(UpdateJobAction.Request request) { if (request.isInternal() == false) { auditor.info(request.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_UPDATED, request.getJobUpdate().getUpdateFields())); @@ -409,32 +402,42 @@ private boolean isJobOpen(ClusterState clusterState, String jobId) { return jobState == JobState.OPENED; } - private ClusterState updateClusterState(Job job, boolean overwrite, ClusterState currentState) { - MlMetadata.Builder builder = createMlMetadataBuilder(currentState); - builder.putJob(job, overwrite); - return buildNewClusterState(currentState, builder); + private Set openJobIds(ClusterState clusterState) { + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + return MlTasks.openJobIds(persistentTasks); } - public void notifyFilterChanged(MlFilter filter, Set addedItems, Set removedItems) { + public void notifyFilterChanged(MlFilter filter, Set addedItems, Set removedItems, + ActionListener updatedListener) { if (addedItems.isEmpty() && removedItems.isEmpty()) { + updatedListener.onResponse(Boolean.TRUE); return; } - ClusterState clusterState = clusterService.state(); - QueryPage jobs = expandJobs("*", true, clusterService.state()); - for (Job job : jobs.results()) { - Set jobFilters = job.getAnalysisConfig().extractReferencedFilters(); - if (jobFilters.contains(filter.getId())) { - if (isJobOpen(clusterState, job.getId())) { - updateJobProcessNotifier.submitJobUpdate(UpdateParams.filterUpdate(job.getId(), filter), - ActionListener.wrap(isUpdated -> { - auditFilterChanges(job.getId(), filter.getId(), addedItems, removedItems); - }, e -> {})); - } else { - auditFilterChanges(job.getId(), filter.getId(), addedItems, removedItems); - } - } - } + jobConfigProvider.findJobsWithCustomRules(ActionListener.wrap( + jobBuilders -> { + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { + for (Job job: jobBuilders) { + Set jobFilters = job.getAnalysisConfig().extractReferencedFilters(); + ClusterState clusterState = clusterService.state(); + if (jobFilters.contains(filter.getId())) { + if (isJobOpen(clusterState, job.getId())) { + updateJobProcessNotifier.submitJobUpdate(UpdateParams.filterUpdate(job.getId(), filter), + ActionListener.wrap(isUpdated -> { + auditFilterChanges(job.getId(), filter.getId(), addedItems, removedItems); + }, e -> { + })); + } else { + auditFilterChanges(job.getId(), filter.getId(), addedItems, removedItems); + } + } + } + + updatedListener.onResponse(Boolean.TRUE); + }); + }, + updatedListener::onFailure + )); } private void auditFilterChanges(String jobId, String filterId, Set addedItems, Set removedItems) { @@ -464,26 +467,40 @@ private static void appendCommaSeparatedSet(Set items, StringBuilder sb) sb.append("]"); } - public void updateProcessOnCalendarChanged(List calendarJobIds) { + public void updateProcessOnCalendarChanged(List calendarJobIds, ActionListener updateListener) { ClusterState clusterState = clusterService.state(); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + Set openJobIds = openJobIds(clusterState); + if (openJobIds.isEmpty()) { + updateListener.onResponse(Boolean.TRUE); + return; + } - List existingJobsOrGroups = - calendarJobIds.stream().filter(mlMetadata::isGroupOrJob).collect(Collectors.toList()); - - Set expandedJobIds = new HashSet<>(); - existingJobsOrGroups.forEach(jobId -> expandedJobIds.addAll(expandJobIds(jobId, true, clusterState))); - for (String jobId : expandedJobIds) { - if (isJobOpen(clusterState, jobId)) { - updateJobProcessNotifier.submitJobUpdate(UpdateParams.scheduledEventsUpdate(jobId), ActionListener.wrap( - isUpdated -> { - if (isUpdated) { - auditor.info(jobId, Messages.getMessage(Messages.JOB_AUDIT_CALENDARS_UPDATED_ON_PROCESS)); + // calendarJobIds may be a group or job + jobConfigProvider.expandGroupIds(calendarJobIds, ActionListener.wrap( + expandedIds -> { + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { + // Merge the expended group members with the request Ids. + // Ids that aren't jobs will be filtered by isJobOpen() + expandedIds.addAll(calendarJobIds); + + for (String jobId : expandedIds) { + if (isJobOpen(clusterState, jobId)) { + updateJobProcessNotifier.submitJobUpdate(UpdateParams.scheduledEventsUpdate(jobId), ActionListener.wrap( + isUpdated -> { + if (isUpdated) { + auditor.info(jobId, Messages.getMessage(Messages.JOB_AUDIT_CALENDARS_UPDATED_ON_PROCESS)); + } + }, + e -> logger.error("[" + jobId + "] failed submitting process update on calendar change", e) + )); } - }, e -> {} - )); - } - } + } + + updateListener.onResponse(Boolean.TRUE); + }); + }, + updateListener::onFailure + )); } public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener actionListener, @@ -515,46 +532,27 @@ public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionList } }; - // Step 1. Do the cluster state update + // Step 1. update the job // ------- - Consumer clusterStateHandler = response -> clusterService.submitStateUpdateTask("revert-snapshot-" + request.getJobId(), - new AckedClusterStateUpdateTask(request, ActionListener.wrap(updateHandler, actionListener::onFailure)) { - - @Override - protected Boolean newResponse(boolean acknowledged) { - if (acknowledged) { - auditor.info(request.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_REVERTED, modelSnapshot.getDescription())); - return true; - } - actionListener.onFailure(new IllegalStateException("Could not revert modelSnapshot on job [" - + request.getJobId() + "], not acknowledged by master.")); - return false; - } - - @Override - public ClusterState execute(ClusterState currentState) { - Job job = getJobOrThrowIfUnknown(request.getJobId(), currentState); - Job.Builder builder = new Job.Builder(job); - builder.setModelSnapshotId(modelSnapshot.getSnapshotId()); - builder.setEstablishedModelMemory(response); - return updateClusterState(builder.build(), true, currentState); - } - }); + Consumer updateJobHandler = response -> { + JobUpdate update = new JobUpdate.Builder(request.getJobId()) + .setModelSnapshotId(modelSnapshot.getSnapshotId()) + .setEstablishedModelMemory(response) + .build(); + + jobConfigProvider.updateJob(request.getJobId(), update, maxModelMemoryLimit, ActionListener.wrap( + job -> { + auditor.info(request.getJobId(), + Messages.getMessage(Messages.JOB_AUDIT_REVERTED, modelSnapshot.getDescription())); + updateHandler.accept(Boolean.TRUE); + }, + actionListener::onFailure + )); + }; // Step 0. Find the appropriate established model memory for the reverted job // ------- - jobResultsProvider.getEstablishedMemoryUsage(request.getJobId(), modelSizeStats.getTimestamp(), modelSizeStats, clusterStateHandler, + jobResultsProvider.getEstablishedMemoryUsage(request.getJobId(), modelSizeStats.getTimestamp(), modelSizeStats, updateJobHandler, actionListener::onFailure); } - - private static MlMetadata.Builder createMlMetadataBuilder(ClusterState currentState) { - return new MlMetadata.Builder(MlMetadata.getMlMetadata(currentState)); - } - - private static ClusterState buildNewClusterState(ClusterState currentState, MlMetadata.Builder builder) { - XPackPlugin.checkReadyForXPackCustomMetadata(currentState); - ClusterState.Builder newState = ClusterState.builder(currentState); - newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build()); - return newState.build(); - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 1b89ecb1250ce..ae13d0371a3a5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.job.persistence; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -35,13 +36,18 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -51,9 +57,11 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -88,7 +96,16 @@ public void putJob(Job job, ActionListener listener) { .setOpType(DocWriteRequest.OpType.CREATE) .request(); - executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + listener::onResponse, + e -> { + if (e instanceof VersionConflictEngineException) { + // the job already exists + listener.onFailure(ExceptionsHelper.jobAlreadyExists(job.getId())); + } else { + listener.onFailure(e); + } + })); } catch (IOException e) { listener.onFailure(new ElasticsearchParseException("Failed to serialise job with id [" + job.getId() + "]", e)); @@ -107,7 +124,7 @@ public void getJob(String jobId, ActionListener jobListener) { GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); - executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener() { @Override public void onResponse(GetResponse getResponse) { if (getResponse.isExists() == false) { @@ -123,7 +140,7 @@ public void onResponse(GetResponse getResponse) { public void onFailure(Exception e) { jobListener.onFailure(e); } - }); + }, client::get); } /** @@ -156,7 +173,7 @@ public void onFailure(Exception e) { } /** - * Get the job and update it by applying {@code jobUpdater} then index the changed job + * Get the job and update it by applying {@code update} then index the changed job * setting the version in the request. Applying the update may cause a validation error * which is returned via {@code updatedJobListener} * @@ -197,26 +214,75 @@ public void onResponse(GetResponse getResponse) { return; } - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - XContentBuilder updatedSource = updatedJob.toXContent(builder, ToXContent.EMPTY_PARAMS); - IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings.DOC_TYPE, Job.documentId(updatedJob.getId())) - .setSource(updatedSource) - .setVersion(version) - .request(); - - executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( - indexResponse -> { - assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; - updatedJobListener.onResponse(updatedJob); - }, - updatedJobListener::onFailure - )); + indexUpdatedJob(updatedJob, version, updatedJobListener); + } - } catch (IOException e) { + @Override + public void onFailure(Exception e) { + updatedJobListener.onFailure(e); + } + }); + } + + /** + * Job update validation function. + * {@code updatedListener} must be called by implementations reporting + * either an validation error or success. + */ + @FunctionalInterface + public interface UpdateValidator { + void validate(Job job, JobUpdate update, ActionListener updatedListener); + } + + /** + * Similar to {@link #updateJob(String, JobUpdate, ByteSizeValue, ActionListener)} but + * with an extra validation step which is called before the updated is applied. + * + * @param jobId The Id of the job to update + * @param update The job update + * @param maxModelMemoryLimit The maximum model memory allowed + * @param validator The job update validator + * @param updatedJobListener Updated job listener + */ + public void updateJobWithValidation(String jobId, JobUpdate update, ByteSizeValue maxModelMemoryLimit, + UpdateValidator validator, ActionListener updatedJobListener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + updatedJobListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + + long version = getResponse.getVersion(); + BytesReference source = getResponse.getSourceAsBytesRef(); + Job originalJob; + try { + originalJob = parseJobLenientlyFromSource(source).build(); + } catch (Exception e) { updatedJobListener.onFailure( - new ElasticsearchParseException("Failed to serialise job with id [" + jobId + "]", e)); + new ElasticsearchParseException("Failed to parse job configuration [" + jobId + "]", e)); + return; } + + validator.validate(originalJob, update, ActionListener.wrap( + validated -> { + Job updatedJob; + try { + // Applying the update may result in a validation error + updatedJob = update.mergeWithJob(originalJob, maxModelMemoryLimit); + } catch (Exception e) { + updatedJobListener.onFailure(e); + return; + } + + indexUpdatedJob(updatedJob, version, updatedJobListener); + }, + updatedJobListener::onFailure + )); } @Override @@ -226,12 +292,70 @@ public void onFailure(Exception e) { }); } + private void indexUpdatedJob(Job updatedJob, long version, ActionListener updatedJobListener) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder updatedSource = updatedJob.toXContent(builder, ToXContent.EMPTY_PARAMS); + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(updatedJob.getId())) + .setSource(updatedSource) + .setVersion(version) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + indexResponse -> { + assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; + updatedJobListener.onResponse(updatedJob); + }, + updatedJobListener::onFailure + )); + + } catch (IOException e) { + updatedJobListener.onFailure( + new ElasticsearchParseException("Failed to serialise job with id [" + updatedJob.getId() + "]", e)); + } + } + + + /** + * Check a job exists. A job exists if it has a configuration document. + * + * If the job does not exist a ResourceNotFoundException is returned to the listener, + * FALSE will never be returned only TRUE or ResourceNotFoundException + * + * @param jobId The jobId to check + * @param listener Exists listener + */ + public void checkJobExists(String jobId, ActionListener listener) { + GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + getRequest.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); + + executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + if (getResponse.isExists() == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + listener.onResponse(Boolean.TRUE); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + /** * Expands an expression into the set of matching names. {@code expresssion} - * may be a wildcard, a job group, a job ID or a list of those. + * may be a wildcard, a job group, a job Id or a list of those. * If {@code expression} == 'ALL', '*' or the empty string then all - * job IDs are returned. - * Job groups are expanded to all the jobs IDs in that group. + * job Ids are returned. + * Job groups are expanded to all the jobs Ids in that group. + * + * If {@code expression} contains a job Id or a Group name then it + * is an error if the job or group do not exist. * * For example, given a set of names ["foo-1", "foo-2", "bar-1", bar-2"], * expressions resolve follows: @@ -249,14 +373,15 @@ public void onFailure(Exception e) { * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception - * @param listener The expanded job IDs listener + * @param listener The expanded job Ids listener */ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); sourceBuilder.sort(Job.ID.getPreferredName()); - String [] includes = new String[] {Job.ID.getPreferredName(), Job.GROUPS.getPreferredName()}; - sourceBuilder.fetchSource(includes, null); + sourceBuilder.fetchSource(false); + sourceBuilder.docValueField(Job.ID.getPreferredName()); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName()); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -271,10 +396,10 @@ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener Set groupsIds = new HashSet<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { - jobIds.add((String)hit.getSourceAsMap().get(Job.ID.getPreferredName())); - List groups = (List)hit.getSourceAsMap().get(Job.GROUPS.getPreferredName()); + jobIds.add(hit.field(Job.ID.getPreferredName()).getValue()); + List groups = hit.field(Job.GROUPS.getPreferredName()).getValues(); if (groups != null) { - groupsIds.addAll(groups); + groupsIds.addAll(groups.stream().map(Object::toString).collect(Collectors.toList())); } } @@ -351,6 +476,75 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener
  • groupIds, ActionListener> listener) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(new TermsQueryBuilder(Job.GROUPS.getPreferredName(), groupIds)); + sourceBuilder.sort(Job.ID.getPreferredName()); + sourceBuilder.fetchSource(false); + sourceBuilder.docValueField(Job.ID.getPreferredName()); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + Set jobIds = new HashSet<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + jobIds.add(hit.field(Job.ID.getPreferredName()).getValue()); + } + + listener.onResponse(jobIds); + }, + listener::onFailure) + , client::search); + } + + public void findJobsWithCustomRules(ActionListener> listener) { + String customRulesPath = Strings.collectionToDelimitedString(Arrays.asList(Job.ANALYSIS_CONFIG.getPreferredName(), + AnalysisConfig.DETECTORS.getPreferredName(), Detector.CUSTOM_RULES_FIELD.getPreferredName()), "."); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.nestedQuery(customRulesPath, QueryBuilders.existsQuery(customRulesPath), ScoreMode.None)) + .size(10000); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + List jobs = new ArrayList<>(); + + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + try { + BytesReference source = hit.getSourceRef(); + Job job = parseJobLenientlyFromSource(source).build(); + jobs.add(job); + } catch (IOException e) { + // TODO A better way to handle this rather than just ignoring the error? + logger.error("Error parsing anomaly detector job configuration [" + hit.getId() + "]", e); + } + } + + listener.onResponse(jobs); + }, + listener::onFailure) + , client::search); + } + private void parseJobLenientlyFromSource(BytesReference source, ActionListener jobListener) { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 8dbc13038c7f7..f135c6f1051ae 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -359,10 +359,20 @@ public void writeUpdateProcessMessage(JobTask jobTask, UpdateParams updateParams updateProcessMessage.setFilter(filter); if (updateParams.isUpdateScheduledEvents()) { - Job job = jobManager.getJobOrThrowIfUnknown(jobTask.getJobId()); - DataCounts dataCounts = getStatistics(jobTask).get().v1(); - ScheduledEventsQueryBuilder query = new ScheduledEventsQueryBuilder().start(job.earliestValidTimestamp(dataCounts)); - jobResultsProvider.scheduledEventsForJob(jobTask.getJobId(), job.getGroups(), query, eventsListener); + jobManager.getJob(jobTask.getJobId(), new ActionListener() { + @Override + public void onResponse(Job job) { + DataCounts dataCounts = getStatistics(jobTask).get().v1(); + ScheduledEventsQueryBuilder query = new ScheduledEventsQueryBuilder() + .start(job.earliestValidTimestamp(dataCounts)); + jobResultsProvider.scheduledEventsForJob(jobTask.getJobId(), job.getGroups(), query, eventsListener); + } + + @Override + public void onFailure(Exception e) { + handler.accept(e); + } + }); } else { eventsListener.onResponse(null); } @@ -393,69 +403,77 @@ public void onFailure(Exception e) { public void openJob(JobTask jobTask, Consumer handler) { String jobId = jobTask.getJobId(); - Job job = jobManager.getJobOrThrowIfUnknown(jobId); - - if (job.getJobVersion() == null) { - handler.accept(ExceptionsHelper.badRequestException("Cannot open job [" + jobId - + "] because jobs created prior to version 5.5 are not supported")); - return; - } - logger.info("Opening job [{}]", jobId); - processByAllocation.putIfAbsent(jobTask.getAllocationId(), new ProcessContext(jobTask)); - jobResultsProvider.getAutodetectParams(job, params -> { - // We need to fork, otherwise we restore model state from a network thread (several GET api calls): - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - handler.accept(e); - } - @Override - protected void doRun() throws Exception { - ProcessContext processContext = processByAllocation.get(jobTask.getAllocationId()); - if (processContext == null) { - logger.debug("Aborted opening job [{}] as it has been closed", jobId); - return; - } - if (processContext.getState() != ProcessContext.ProcessStateName.NOT_RUNNING) { - logger.debug("Cannot open job [{}] when its state is [{}]", jobId, processContext.getState().getClass().getName()); + jobManager.getJob(jobId, ActionListener.wrap( + // NORELEASE JIndex. Should not be doing this work on the network thread + job -> { + if (job.getJobVersion() == null) { + handler.accept(ExceptionsHelper.badRequestException("Cannot open job [" + jobId + + "] because jobs created prior to version 5.5 are not supported")); return; } - try { - createProcessAndSetRunning(processContext, params, handler); - processContext.getAutodetectCommunicator().init(params.modelSnapshot()); - setJobState(jobTask, JobState.OPENED); - } catch (Exception e1) { - // No need to log here as the persistent task framework will log it - try { - // Don't leave a partially initialised process hanging around - processContext.newKillBuilder() - .setAwaitCompletion(false) - .setFinish(false) - .kill(); - processByAllocation.remove(jobTask.getAllocationId()); - } finally { - setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); - } - } - } - }); - }, e1 -> { - logger.warn("Failed to gather information required to open job [" + jobId + "]", e1); - setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); - }); + + processByAllocation.putIfAbsent(jobTask.getAllocationId(), new ProcessContext(jobTask)); + jobResultsProvider.getAutodetectParams(job, params -> { + // We need to fork, otherwise we restore model state from a network thread (several GET api calls): + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + handler.accept(e); + } + + @Override + protected void doRun() { + ProcessContext processContext = processByAllocation.get(jobTask.getAllocationId()); + if (processContext == null) { + logger.debug("Aborted opening job [{}] as it has been closed", jobId); + return; + } + if (processContext.getState() != ProcessContext.ProcessStateName.NOT_RUNNING) { + logger.debug("Cannot open job [{}] when its state is [{}]", + jobId, processContext.getState().getClass().getName()); + return; + } + + try { + createProcessAndSetRunning(processContext, job, params, handler); + processContext.getAutodetectCommunicator().init(params.modelSnapshot()); + setJobState(jobTask, JobState.OPENED); + } catch (Exception e1) { + // No need to log here as the persistent task framework will log it + try { + // Don't leave a partially initialised process hanging around + processContext.newKillBuilder() + .setAwaitCompletion(false) + .setFinish(false) + .kill(); + processByAllocation.remove(jobTask.getAllocationId()); + } finally { + setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); + } + } + } + }); + }, e1 -> { + logger.warn("Failed to gather information required to open job [" + jobId + "]", e1); + setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); + }); + }, + handler + )); + } - private void createProcessAndSetRunning(ProcessContext processContext, AutodetectParams params, Consumer handler) { + private void createProcessAndSetRunning(ProcessContext processContext, Job job, AutodetectParams params, Consumer handler) { // At this point we lock the process context until the process has been started. // The reason behind this is to ensure closing the job does not happen before // the process is started as that can result to the job getting seemingly closed // but the actual process is hanging alive. processContext.tryLock(); try { - AutodetectCommunicator communicator = create(processContext.getJobTask(), params, handler); + AutodetectCommunicator communicator = create(processContext.getJobTask(), job, params, handler); processContext.setRunning(communicator); } finally { // Now that the process is running and we have updated its state we can unlock. @@ -465,7 +483,7 @@ private void createProcessAndSetRunning(ProcessContext processContext, Autodetec } } - AutodetectCommunicator create(JobTask jobTask, AutodetectParams autodetectParams, Consumer handler) { + AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodetectParams, Consumer handler) { // Closing jobs can still be using some or all threads in MachineLearning.AUTODETECT_THREAD_POOL_NAME // that an open job uses, so include them too when considering if enough threads are available. int currentRunningJobs = processByAllocation.size(); @@ -490,7 +508,6 @@ AutodetectCommunicator create(JobTask jobTask, AutodetectParams autodetectParams } } - Job job = jobManager.getJobOrThrowIfUnknown(jobId); // A TP with no queue, so that we fail immediately if there are no threads available ExecutorService autoDetectExecutorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME); DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java index 75e79ede014d4..29108f46c72c1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java @@ -88,6 +88,7 @@ public void testMachineLearningPutJobActionRestricted() { } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningOpenJobActionRestricted() throws Exception { String jobId = "testmachinelearningopenjobactionrestricted"; assertMLAllowed(true); @@ -139,6 +140,7 @@ public void testMachineLearningOpenJobActionRestricted() throws Exception { } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningPutDatafeedActionRestricted() throws Exception { String jobId = "testmachinelearningputdatafeedactionrestricted"; String datafeedId = jobId + "-datafeed"; @@ -186,6 +188,7 @@ public void testMachineLearningPutDatafeedActionRestricted() throws Exception { } } + @AwaitsFix(bugUrl = "JIndex development") public void testAutoCloseJobWithDatafeed() throws Exception { String jobId = "testautoclosejobwithdatafeed"; String datafeedId = jobId + "-datafeed"; @@ -288,6 +291,7 @@ public void testAutoCloseJobWithDatafeed() throws Exception { }); } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningStartDatafeedActionRestricted() throws Exception { String jobId = "testmachinelearningstartdatafeedactionrestricted"; String datafeedId = jobId + "-datafeed"; @@ -362,6 +366,7 @@ public void testMachineLearningStartDatafeedActionRestricted() throws Exception } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningStopDatafeedActionNotRestricted() throws Exception { String jobId = "testmachinelearningstopdatafeedactionnotrestricted"; String datafeedId = jobId + "-datafeed"; @@ -428,6 +433,7 @@ public void testMachineLearningStopDatafeedActionNotRestricted() throws Exceptio } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningCloseJobActionNotRestricted() throws Exception { String jobId = "testmachinelearningclosejobactionnotrestricted"; assertMLAllowed(true); @@ -471,6 +477,7 @@ public void testMachineLearningCloseJobActionNotRestricted() throws Exception { } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningDeleteJobActionNotRestricted() throws Exception { String jobId = "testmachinelearningclosejobactionnotrestricted"; assertMLAllowed(true); @@ -496,6 +503,7 @@ public void testMachineLearningDeleteJobActionNotRestricted() throws Exception { } } + @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningDeleteDatafeedActionNotRestricted() throws Exception { String jobId = "testmachinelearningdeletedatafeedactionnotrestricted"; String datafeedId = jobId + "-datafeed"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java index 181636de13663..6c14423d9acdb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java @@ -78,7 +78,7 @@ protected T blockingCall(Consumer> function) throws Except AtomicReference responseHolder = new AtomicReference<>(); blockingCall(function, responseHolder, exceptionHolder); if (exceptionHolder.get() != null) { - assertNotNull(exceptionHolder.get().getMessage(), exceptionHolder.get()); + assertNull(exceptionHolder.get().getMessage(), exceptionHolder.get()); } return responseHolder.get(); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java index 687292b3c85d4..53bdfbdcb3b69 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java @@ -15,6 +15,9 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; + public class MlTasksTests extends ESTestCase { public void testGetJobState() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); @@ -65,4 +68,19 @@ public void testGetDatafeedTask() { assertNotNull(MlTasks.getDatafeedTask("foo", tasksBuilder.build())); assertNull(MlTasks.getDatafeedTask("other", tasksBuilder.build())); } + + public void testOpenJobIds() { + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + assertThat(MlTasks.openJobIds(tasksBuilder.build()), empty()); + + tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo-1"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.jobTaskId("bar"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("bar"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.datafeedTaskId("df"), StartDatafeedAction.TASK_NAME, + new StartDatafeedAction.DatafeedParams("df", 0L), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + + assertThat(MlTasks.openJobIds(tasksBuilder.build()), containsInAnyOrder("foo-1", "bar")); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index d85d8e1d8cbcd..712206e37556a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -6,19 +6,21 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; @@ -36,7 +38,9 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsInstanceOf.instanceOf; public class JobConfigProviderIT extends MlSingleNodeTestCase { @@ -60,6 +64,29 @@ public void testGetMissingJob() throws InterruptedException { assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); } + public void testCheckJobExists() throws InterruptedException { + AtomicReference jobExistsHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(actionListener -> jobConfigProvider.checkJobExists("missing", actionListener), jobExistsHolder, exceptionHolder); + + assertNull(jobExistsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + + AtomicReference indexResponseHolder = new AtomicReference<>(); + + // Create job + Job job = createJob("existing-job", null).build(new Date()); + blockingCall(actionListener -> jobConfigProvider.putJob(job, actionListener), indexResponseHolder, exceptionHolder); + + exceptionHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.checkJobExists("existing-job", actionListener), jobExistsHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertNotNull(jobExistsHolder.get()); + assertTrue(jobExistsHolder.get()); + } + public void testOverwriteNotAllowed() throws InterruptedException { final String jobId = "same-id"; @@ -77,7 +104,8 @@ public void testOverwriteNotAllowed() throws InterruptedException { blockingCall(actionListener -> jobConfigProvider.putJob(jobWithSameId, actionListener), indexResponseHolder, exceptionHolder); assertNull(indexResponseHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(VersionConflictEngineException.class)); + assertThat(exceptionHolder.get(), instanceOf(ResourceAlreadyExistsException.class)); + assertEquals("The job cannot be created with the Id 'same-id'. The Id is already used.", exceptionHolder.get().getMessage()); } public void testCrud() throws InterruptedException { @@ -163,6 +191,46 @@ public void testUpdateWithAValidationError() throws Exception { assertThat(exceptionHolder.get().getMessage(), containsString("Invalid detector rule:")); } + public void testUpdateWithValidator() throws Exception { + final String jobId = "job-update-with-validator"; + + // Create job + Job newJob = createJob(jobId, null).build(new Date()); + this.blockingCall(actionListener -> jobConfigProvider.putJob(newJob, actionListener)); + + JobUpdate jobUpdate = new JobUpdate.Builder(jobId).setDescription("This job has been updated").build(); + + JobConfigProvider.UpdateValidator validator = (job, update, listener) -> { + listener.onResponse(null); + }; + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference updateJobResponseHolder = new AtomicReference<>(); + // update with the no-op validator + blockingCall(actionListener -> + jobConfigProvider.updateJobWithValidation(jobId, jobUpdate, new ByteSizeValue(32), validator, actionListener), + updateJobResponseHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertNotNull(updateJobResponseHolder.get()); + assertEquals("This job has been updated", updateJobResponseHolder.get().getDescription()); + + JobConfigProvider.UpdateValidator validatorWithAnError = (job, update, listener) -> { + listener.onFailure(new IllegalStateException("I don't like this update")); + }; + + updateJobResponseHolder.set(null); + // Update with a validator that errors + blockingCall(actionListener -> jobConfigProvider.updateJobWithValidation(jobId, jobUpdate, new ByteSizeValue(32), + validatorWithAnError, actionListener), + updateJobResponseHolder, exceptionHolder); + + assertNull(updateJobResponseHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(IllegalStateException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("I don't like this update")); + } + public void testAllowNoJobs() throws InterruptedException { AtomicReference> jobIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); @@ -297,7 +365,61 @@ public void testExpandJobs_WildCardExpansion() throws Exception { assertThat(expandedJobs, containsInAnyOrder(bar1)); } - private Job.Builder createJob(String jobId, List groups) { + public void testExpandGroups() throws Exception { + putJob(createJob("apples", Collections.singletonList("fruit"))); + putJob(createJob("pears", Collections.singletonList("fruit"))); + putJob(createJob("broccoli", Collections.singletonList("veg"))); + putJob(createJob("potato", Collections.singletonList("veg"))); + putJob(createJob("tomato", Arrays.asList("fruit", "veg"))); + putJob(createJob("unrelated", Collections.emptyList())); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + Set expandedIds = blockingCall(actionListener -> + jobConfigProvider.expandGroupIds(Collections.singletonList("fruit"), actionListener)); + assertThat(expandedIds, containsInAnyOrder("apples", "pears", "tomato")); + + expandedIds = blockingCall(actionListener -> + jobConfigProvider.expandGroupIds(Collections.singletonList("veg"), actionListener)); + assertThat(expandedIds, containsInAnyOrder("broccoli", "potato", "tomato")); + + expandedIds = blockingCall(actionListener -> + jobConfigProvider.expandGroupIds(Arrays.asList("fruit", "veg"), actionListener)); + assertThat(expandedIds, containsInAnyOrder("apples", "pears", "broccoli", "potato", "tomato")); + + expandedIds = blockingCall(actionListener -> + jobConfigProvider.expandGroupIds(Collections.singletonList("unknown-group"), actionListener)); + assertThat(expandedIds, empty()); + } + + public void testFindJobsWithCustomRules_GivenNoJobs() throws Exception { + List foundJobs = blockingCall(listener -> jobConfigProvider.findJobsWithCustomRules(listener)); + assertThat(foundJobs.isEmpty(), is(true)); + } + + public void testFindJobsWithCustomRules() throws Exception { + putJob(createJob("job-without-rules", Collections.emptyList())); + + DetectionRule rule = new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 0.0))).build(); + + Job.Builder jobWithRules1 = createJob("job-with-rules-1", Collections.emptyList()); + jobWithRules1 = addCustomRule(jobWithRules1, rule); + putJob(jobWithRules1); + Job.Builder jobWithRules2 = createJob("job-with-rules-2", Collections.emptyList()); + jobWithRules2 = addCustomRule(jobWithRules2, rule); + putJob(jobWithRules2); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + List foundJobs = blockingCall(listener -> jobConfigProvider.findJobsWithCustomRules(listener)); + + Set foundJobIds = foundJobs.stream().map(Job::getId).collect(Collectors.toSet()); + assertThat(foundJobIds.size(), equalTo(2)); + assertThat(foundJobIds, containsInAnyOrder(jobWithRules1.getId(), jobWithRules2.getId())); + } + + private static Job.Builder createJob(String jobId, List groups) { Detector.Builder d1 = new Detector.Builder("info_content", "domain"); d1.setOverFieldName("client"); AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d1.build())); @@ -312,6 +434,13 @@ private Job.Builder createJob(String jobId, List groups) { return builder; } + private static Job.Builder addCustomRule(Job.Builder job, DetectionRule rule) { + JobUpdate.Builder update1 = new JobUpdate.Builder(job.getId()); + update1.setDetectorUpdates(Collections.singletonList(new JobUpdate.DetectorUpdate(0, null, Collections.singletonList(rule)))); + Job updatedJob = update1.build().mergeWithJob(job.build(new Date()), null); + return new Job.Builder(updatedJob); + } + private Job putJob(Job.Builder job) throws Exception { Job builtJob = job.build(new Date()); this.blockingCall(actionListener -> jobConfigProvider.putJob(builtJob, actionListener)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index a9162cb2ae4df..98c406694ec5c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -8,19 +8,25 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PutJobAction; @@ -33,8 +39,11 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.RuleScope; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; +import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.Before; @@ -43,16 +52,24 @@ import org.mockito.Mockito; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.TreeSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests.addJobTask; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.mockito.Matchers.any; @@ -66,8 +83,8 @@ public class JobManagerTests extends ESTestCase { private Environment environment; private AnalysisRegistry analysisRegistry; - private Client client; private ClusterService clusterService; + private ThreadPool threadPool; private JobResultsProvider jobResultsProvider; private Auditor auditor; private UpdateJobProcessNotifier updateJobProcessNotifier; @@ -77,47 +94,134 @@ public void setup() throws Exception { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); environment = TestEnvironment.newEnvironment(settings); analysisRegistry = CategorizationAnalyzerTests.buildTestAnalysisRegistry(environment); - client = mock(Client.class); clusterService = mock(ClusterService.class); + jobResultsProvider = mock(JobResultsProvider.class); auditor = mock(Auditor.class); updateJobProcessNotifier = mock(UpdateJobProcessNotifier.class); + + ExecutorService executorService = mock(ExecutorService.class); + threadPool = mock(ThreadPool.class); + org.elasticsearch.mock.orig.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); } - public void testGetJobOrThrowIfUnknown_GivenUnknownJob() { - ClusterState cs = createClusterState(); - ESTestCase.expectThrows(ResourceNotFoundException.class, () -> JobManager.getJobOrThrowIfUnknown("foo", cs)); + public void testGetJobNotInIndexOrCluster() { + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + when(clusterService.state()).thenReturn(clusterState); + + // job document does not exist + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(false); + MockClientBuilder mockClientBuilder = new MockClientBuilder("jm-test"); + mockClientBuilder.get(getResponse); + + JobManager jobManager = createJobManager(mockClientBuilder.build()); + + AtomicReference exceptionHolder = new AtomicReference<>(); + jobManager.getJob("non-job", ActionListener.wrap( + job -> fail("Job not expected"), + e -> exceptionHolder.set(e) + )); + + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); } - public void testGetJobOrThrowIfUnknown_GivenKnownJob() { - Job job = buildJobBuilder("foo").build(); - MlMetadata mlMetadata = new MlMetadata.Builder().putJob(job, false).build(); - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)).build(); + public void testGetJobFromClusterWhenNotInIndex() { + String clusterJobId = "cluster-job"; + Job clusterJob = buildJobBuilder(clusterJobId).build(); + + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + mlMetadata.putJob(clusterJob, false); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + when(clusterService.state()).thenReturn(clusterState); + + // job document does not exist + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(false); + MockClientBuilder mockClientBuilder = new MockClientBuilder("jm-test"); + mockClientBuilder.get(getResponse); + + JobManager jobManager = createJobManager(mockClientBuilder.build()); - assertEquals(job, JobManager.getJobOrThrowIfUnknown("foo", cs)); + AtomicReference jobHolder = new AtomicReference<>(); + jobManager.getJob(clusterJobId, ActionListener.wrap( + job -> jobHolder.set(job), + e -> fail(e.getMessage()) + )); + + assertNotNull(jobHolder.get()); + assertEquals(clusterJob, jobHolder.get()); } - public void testExpandJobs_GivenAll() { + public void testExpandJobsFromClusterStateAndIndex() throws IOException { + Job csJobFoo1 = buildJobBuilder("foo-cs-1").build(); + Job csJobFoo2 = buildJobBuilder("foo-cs-2").build(); + Job csJobBar = buildJobBuilder("bar-cs").build(); + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - for (int i = 0; i < 3; i++) { - mlMetadata.putJob(buildJobBuilder(Integer.toString(i)).build(), false); - } + mlMetadata.putJob(csJobFoo1, false); + mlMetadata.putJob(csJobFoo2, false); + mlMetadata.putJob(csJobBar, false); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata.build())).build(); + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + when(clusterService.state()).thenReturn(clusterState); + + + List docsAsBytes = new ArrayList<>(); + + Job.Builder indexJobFoo = buildJobBuilder("foo-index"); + docsAsBytes.add(toBytesReference(indexJobFoo.build())); + + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); + + + AtomicReference> jobsHolder = new AtomicReference<>(); + jobManager.expandJobs("_all", true, ActionListener.wrap( + jobs -> jobsHolder.set(jobs), + e -> fail(e.getMessage()) + )); + + assertNotNull(jobsHolder.get()); + assertThat(jobsHolder.get().results(), hasSize(4)); + List jobIds = jobsHolder.get().results().stream().map(Job::getId).collect(Collectors.toList()); + assertThat(jobIds, contains("bar-cs", "foo-cs-1", "foo-cs-2", "foo-index")); - JobManager jobManager = createJobManager(); - QueryPage result = jobManager.expandJobs("_all", true, clusterState); + jobsHolder.set(null); + jobManager.expandJobs("foo*", true, ActionListener.wrap( + jobs -> jobsHolder.set(jobs), + e -> fail(e.getMessage()) + )); - assertThat(result.count(), equalTo(3L)); - assertThat(result.results().get(0).getId(), equalTo("0")); - assertThat(result.results().get(1).getId(), equalTo("1")); - assertThat(result.results().get(2).getId(), equalTo("2")); + assertNotNull(jobsHolder.get()); + assertThat(jobsHolder.get().results(), hasSize(3)); + jobIds = jobsHolder.get().results().stream().map(Job::getId).collect(Collectors.toList()); + assertThat(jobIds, contains("foo-cs-1", "foo-cs-2", "foo-index")); } @SuppressWarnings("unchecked") public void testPutJob_AddsCreateTime() throws IOException { - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + JobManager jobManager = createJobManager(mockClientBuilder.build()); + PutJobAction.Request putJobRequest = new PutJobAction.Request(createJob()); doAnswer(invocation -> { @@ -153,8 +257,10 @@ public void onFailure(Exception e) { }); } - public void testPutJob_ThrowsIfJobExists() throws IOException { - JobManager jobManager = createJobManager(); + public void testPutJob_ThrowsIfJobExistsInClusterState() throws IOException { + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + JobManager jobManager = createJobManager(mockClientBuilder.build()); + PutJobAction.Request putJobRequest = new PutJobAction.Request(createJob()); MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); @@ -177,14 +283,19 @@ public void onFailure(Exception e) { public void testNotifyFilterChangedGivenNoop() { MlFilter filter = MlFilter.builder("my_filter").build(); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + JobManager jobManager = createJobManager(mockClientBuilder.build()); - jobManager.notifyFilterChanged(filter, Collections.emptySet(), Collections.emptySet()); + jobManager.notifyFilterChanged(filter, Collections.emptySet(), Collections.emptySet(), ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - public void testNotifyFilterChanged() { + @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") + public void testNotifyFilterChanged() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); DetectionRule filterRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "foo_filter")).build(); @@ -192,19 +303,21 @@ public void testNotifyFilterChanged() { AnalysisConfig.Builder filterAnalysisConfig = new AnalysisConfig.Builder(Collections.singletonList( detectorReferencingFilter.build())); + List docsAsBytes = new ArrayList<>(); + Job.Builder jobReferencingFilter1 = buildJobBuilder("job-referencing-filter-1"); jobReferencingFilter1.setAnalysisConfig(filterAnalysisConfig); + docsAsBytes.add(toBytesReference(jobReferencingFilter1.build())); + Job.Builder jobReferencingFilter2 = buildJobBuilder("job-referencing-filter-2"); jobReferencingFilter2.setAnalysisConfig(filterAnalysisConfig); + docsAsBytes.add(toBytesReference(jobReferencingFilter2.build())); + Job.Builder jobReferencingFilter3 = buildJobBuilder("job-referencing-filter-3"); - jobReferencingFilter3.setAnalysisConfig(filterAnalysisConfig); - Job.Builder jobWithoutFilter = buildJobBuilder("job-without-filter"); + jobReferencingFilter2.setAnalysisConfig(filterAnalysisConfig); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - mlMetadata.putJob(jobReferencingFilter1.build(), false); - mlMetadata.putJob(jobReferencingFilter2.build(), false); - mlMetadata.putJob(jobReferencingFilter3.build(), false); - mlMetadata.putJob(jobWithoutFilter.build(), false); + Job.Builder jobWithoutFilter = buildJobBuilder("job-without-filter"); + docsAsBytes.add(toBytesReference(jobWithoutFilter.build())); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(jobReferencingFilter1.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -213,8 +326,7 @@ public void testNotifyFilterChanged() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); when(clusterService.state()).thenReturn(clusterState); @@ -224,12 +336,17 @@ public void testNotifyFilterChanged() { return null; }).when(updateJobProcessNotifier).submitJobUpdate(any(), any()); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); MlFilter filter = MlFilter.builder("foo_filter").setItems("a", "b").build(); jobManager.notifyFilterChanged(filter, new TreeSet<>(Arrays.asList("item 1", "item 2")), - new TreeSet<>(Collections.singletonList("item 3"))); + new TreeSet<>(Collections.singletonList("item 3")), ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); ArgumentCaptor updateParamsCaptor = ArgumentCaptor.forClass(UpdateParams.class); verify(updateJobProcessNotifier, times(2)).submitJobUpdate(updateParamsCaptor.capture(), any(ActionListener.class)); @@ -250,7 +367,8 @@ public void testNotifyFilterChanged() { Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - public void testNotifyFilterChangedGivenOnlyAddedItems() { + @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") + public void testNotifyFilterChangedGivenOnlyAddedItems() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); DetectionRule filterRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "foo_filter")).build(); @@ -261,26 +379,33 @@ public void testNotifyFilterChangedGivenOnlyAddedItems() { Job.Builder jobReferencingFilter = buildJobBuilder("job-referencing-filter"); jobReferencingFilter.setAnalysisConfig(filterAnalysisConfig); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - mlMetadata.putJob(jobReferencingFilter.build(), false); + List docsAsBytes = Collections.singletonList(toBytesReference(jobReferencingFilter.build())); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); when(clusterService.state()).thenReturn(clusterState); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); MlFilter filter = MlFilter.builder("foo_filter").build(); - jobManager.notifyFilterChanged(filter, new TreeSet<>(Arrays.asList("a", "b")), Collections.emptySet()); + jobManager.notifyFilterChanged(filter, new TreeSet<>(Arrays.asList("a", "b")), Collections.emptySet(), + ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); verify(auditor).info(jobReferencingFilter.getId(), "Filter [foo_filter] has been modified; added items: ['a', 'b']"); Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - public void testNotifyFilterChangedGivenOnlyRemovedItems() { + @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") + public void testNotifyFilterChangedGivenOnlyRemovedItems() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); DetectionRule filterRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "foo_filter")).build(); @@ -290,37 +415,42 @@ public void testNotifyFilterChangedGivenOnlyRemovedItems() { Job.Builder jobReferencingFilter = buildJobBuilder("job-referencing-filter"); jobReferencingFilter.setAnalysisConfig(filterAnalysisConfig); + List docsAsBytes = Collections.singletonList(toBytesReference(jobReferencingFilter.build())); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - mlMetadata.putJob(jobReferencingFilter.build(), false); - + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); when(clusterService.state()).thenReturn(clusterState); + when(clusterService.state()).thenReturn(clusterState); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); MlFilter filter = MlFilter.builder("foo_filter").build(); - jobManager.notifyFilterChanged(filter, Collections.emptySet(), new TreeSet<>(Arrays.asList("a", "b"))); + jobManager.notifyFilterChanged(filter, Collections.emptySet(), new TreeSet<>(Arrays.asList("a", "b")), + ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); verify(auditor).info(jobReferencingFilter.getId(), "Filter [foo_filter] has been modified; removed items: ['a', 'b']"); Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - public void testUpdateProcessOnCalendarChanged() { + public void testUpdateProcessOnCalendarChanged() throws IOException { + List docsAsBytes = new ArrayList<>(); Job.Builder job1 = buildJobBuilder("job-1"); + docsAsBytes.add(toBytesReference(job1.build())); Job.Builder job2 = buildJobBuilder("job-2"); +// docsAsBytes.add(toBytesReference(job2.build())); Job.Builder job3 = buildJobBuilder("job-3"); + docsAsBytes.add(toBytesReference(job3.build())); Job.Builder job4 = buildJobBuilder("job-4"); - - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - mlMetadata.putJob(job1.build(), false); - mlMetadata.putJob(job2.build(), false); - mlMetadata.putJob(job3.build(), false); - mlMetadata.putJob(job4.build(), false); + docsAsBytes.add(toBytesReference(job4.build())); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job1.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -329,14 +459,19 @@ public void testUpdateProcessOnCalendarChanged() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); when(clusterService.state()).thenReturn(clusterState); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); - jobManager.updateProcessOnCalendarChanged(Arrays.asList("job-1", "job-3", "job-4")); + jobManager.updateProcessOnCalendarChanged(Arrays.asList("job-1", "job-3", "job-4"), + ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); ArgumentCaptor updateParamsCaptor = ArgumentCaptor.forClass(UpdateParams.class); verify(updateJobProcessNotifier, times(2)).submitJobUpdate(updateParamsCaptor.capture(), any(ActionListener.class)); @@ -349,17 +484,17 @@ public void testUpdateProcessOnCalendarChanged() { assertThat(capturedUpdateParams.get(1).isUpdateScheduledEvents(), is(true)); } - public void testUpdateProcessOnCalendarChanged_GivenGroups() { + public void testUpdateProcessOnCalendarChanged_GivenGroups() throws IOException { Job.Builder job1 = buildJobBuilder("job-1"); job1.setGroups(Collections.singletonList("group-1")); Job.Builder job2 = buildJobBuilder("job-2"); job2.setGroups(Collections.singletonList("group-1")); Job.Builder job3 = buildJobBuilder("job-3"); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - mlMetadata.putJob(job1.build(), false); - mlMetadata.putJob(job2.build(), false); - mlMetadata.putJob(job3.build(), false); + List docsAsBytes = new ArrayList<>(); + docsAsBytes.add(toBytesReference(job1.build())); + docsAsBytes.add(toBytesReference(job2.build())); +// docsAsBytes.add(toBytesReference(job3.build())); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job1.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -368,14 +503,19 @@ public void testUpdateProcessOnCalendarChanged_GivenGroups() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); when(clusterService.state()).thenReturn(clusterState); - JobManager jobManager = createJobManager(); + MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); + mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + JobManager jobManager = createJobManager(mockClientBuilder.build()); - jobManager.updateProcessOnCalendarChanged(Collections.singletonList("group-1")); + jobManager.updateProcessOnCalendarChanged(Collections.singletonList("group-1"), + ActionListener.wrap( + r -> {}, + e -> fail(e.getMessage()) + )); ArgumentCaptor updateParamsCaptor = ArgumentCaptor.forClass(UpdateParams.class); verify(updateJobProcessNotifier, times(2)).submitJobUpdate(updateParamsCaptor.capture(), any(ActionListener.class)); @@ -400,12 +540,12 @@ private Job.Builder createJob() { return builder; } - private JobManager createJobManager() { + private JobManager createJobManager(Client client) { ClusterSettings clusterSettings = new ClusterSettings(environment.settings(), Collections.singleton(MachineLearningField.MAX_MODEL_MEMORY_LIMIT)); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); return new JobManager(environment, environment.settings(), jobResultsProvider, clusterService, - auditor, client, updateJobProcessNotifier); + auditor, threadPool, client, updateJobProcessNotifier); } private ClusterState createClusterState() { @@ -413,4 +553,11 @@ private ClusterState createClusterState() { builder.metaData(MetaData.builder()); return builder.build(); } + + private BytesReference toBytesReference(ToXContent content) throws IOException { + try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) { + content.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(xContentBuilder); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 7dbe3bbf1ffd8..a5f3d5ff5179c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; @@ -39,11 +40,14 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.threadpool.ThreadPool; @@ -52,6 +56,7 @@ import org.mockito.stubbing.Answer; import java.io.IOException; +import java.util.List; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertArrayEquals; @@ -164,6 +169,19 @@ public MockClientBuilder prepareGet(String index, String type, String id, GetRes return this; } + public MockClientBuilder get(GetResponse response) { + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocationOnMock) { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(response); + return null; + } + }).when(client).get(any(), any()); + + return this; + } + public MockClientBuilder prepareCreate(String index) { CreateIndexRequestBuilder createIndexRequestBuilder = mock(CreateIndexRequestBuilder.class); CreateIndexResponse response = mock(CreateIndexResponse.class); @@ -250,6 +268,46 @@ public MockClientBuilder prepareSearch(String index, String type, int from, int return this; } + /** + * Creates a {@link SearchResponse} with a {@link SearchHit} for each element of {@code docs} + * @param indexName Index being searched + * @param docs Returned in the SearchResponse + * @return + */ + @SuppressWarnings("unchecked") + public MockClientBuilder prepareSearch(String indexName, List docs) { + SearchRequestBuilder builder = mock(SearchRequestBuilder.class); + when(builder.setIndicesOptions(any())).thenReturn(builder); + when(builder.setQuery(any())).thenReturn(builder); + when(builder.setSource(any())).thenReturn(builder); + SearchRequest request = new SearchRequest(indexName); + when(builder.request()).thenReturn(request); + + when(client.prepareSearch(eq(indexName))).thenReturn(builder); + + SearchHit hits [] = new SearchHit[docs.size()]; + for (int i=0; i() { + @Override + public Void answer(InvocationOnMock invocationOnMock) { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(response); + return null; + } + }).when(client).search(eq(request), any()); + + return this; + } + public MockClientBuilder prepareSearchAnySize(String index, String type, SearchResponse response, ArgumentCaptor filter) { SearchRequestBuilder builder = mock(SearchRequestBuilder.class); when(builder.setTypes(eq(type))).thenReturn(builder); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 35a8bfae48173..96c3176272f46 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -39,8 +39,8 @@ import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; @@ -126,8 +126,13 @@ public void setup() throws Exception { normalizerFactory = mock(NormalizerFactory.class); auditor = mock(Auditor.class); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(createJobDetails("foo")); + return null; + }).when(jobManager).getJob(eq("foo"), any()); - when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") Consumer handler = (Consumer) invocationOnMock.getArguments()[1]; @@ -167,6 +172,27 @@ public void testMaxOpenJobsSetting_givenOldAndNewSettings() { + "See the breaking changes documentation for the next major version."); } + public void testOpenJob() { + Client client = mock(Client.class); + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(createJobDetails("foo")); + return null; + }).when(jobManager).getJob(eq("foo"), any()); + AutodetectProcessManager manager = createManager(communicator, client); + + JobTask jobTask = mock(JobTask.class); + when(jobTask.getJobId()).thenReturn("foo"); + when(jobTask.getAllocationId()).thenReturn(1L); + manager.openJob(jobTask, e -> {}); + assertEquals(1, manager.numberOfOpenJobs()); + assertTrue(manager.jobHasActiveAutodetectProcess(jobTask)); + verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L)), any()); + } + + public void testOpenJob_withoutVersion() { Client client = mock(Client.class); AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); @@ -175,41 +201,33 @@ public void testOpenJob_withoutVersion() { Job job = jobBuilder.build(); assertThat(job.getJobVersion(), is(nullValue())); - when(jobManager.getJobOrThrowIfUnknown(job.getId())).thenReturn(job); - AutodetectProcessManager manager = createManager(communicator, client); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(job); + return null; + }).when(jobManager).getJob(eq(job.getId()), any()); + AutodetectProcessManager manager = createManager(communicator, client); JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn(job.getId()); - AtomicReference errorHolder = new AtomicReference<>(); manager.openJob(jobTask, errorHolder::set); - Exception error = errorHolder.get(); assertThat(error, is(notNullValue())); assertThat(error.getMessage(), equalTo("Cannot open job [no_version] because jobs created prior to version 5.5 are not supported")); } - public void testOpenJob() { - Client client = mock(Client.class); - AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); - when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); - AutodetectProcessManager manager = createManager(communicator, client); - - JobTask jobTask = mock(JobTask.class); - when(jobTask.getJobId()).thenReturn("foo"); - when(jobTask.getAllocationId()).thenReturn(1L); - manager.openJob(jobTask, e -> {}); - assertEquals(1, manager.numberOfOpenJobs()); - assertTrue(manager.jobHasActiveAutodetectProcess(jobTask)); - verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L)), any()); - } - @SuppressWarnings("unchecked") public void testOpenJob_exceedMaxNumJobs() { - when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); - when(jobManager.getJobOrThrowIfUnknown("bar")).thenReturn(createJobDetails("bar")); - when(jobManager.getJobOrThrowIfUnknown("baz")).thenReturn(createJobDetails("baz")); - when(jobManager.getJobOrThrowIfUnknown("foobar")).thenReturn(createJobDetails("foobar")); + for (String jobId : new String [] {"foo", "bar", "baz", "foobar"}) { + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(createJobDetails(jobId)); + return null; + }).when(jobManager).getJob(eq(jobId), any()); + } Client client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); @@ -578,7 +596,14 @@ public void testCreate_notEnoughThreads() throws IOException { doThrow(new EsRejectedExecutionException("")).when(executorService).submit(any(Runnable.class)); when(threadPool.executor(anyString())).thenReturn(executorService); when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(mock(ThreadPool.Cancellable.class)); - when(jobManager.getJobOrThrowIfUnknown("my_id")).thenReturn(createJobDetails("my_id")); + Job job = createJobDetails("my_id"); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(job); + return null; + }).when(jobManager).getJob(eq("my_id"), any()); + AutodetectProcess autodetectProcess = mock(AutodetectProcess.class); AutodetectProcessFactory autodetectProcessFactory = (j, autodetectParams, e, onProcessCrash) -> autodetectProcess; @@ -589,7 +614,7 @@ public void testCreate_notEnoughThreads() throws IOException { JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("my_id"); expectThrows(EsRejectedExecutionException.class, - () -> manager.create(jobTask, buildAutodetectParams(), e -> {})); + () -> manager.create(jobTask, job, buildAutodetectParams(), e -> {})); verify(autodetectProcess, times(1)).close(); } @@ -599,7 +624,7 @@ public void testCreate_givenFirstTime() { JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("foo"); - manager.create(jobTask, buildAutodetectParams(), e -> {}); + manager.create(jobTask, createJobDetails("foo"), buildAutodetectParams(), e -> {}); String expectedNotification = "Loading model snapshot [N/A], job latest_record_timestamp [N/A]"; verify(auditor).info("foo", expectedNotification); @@ -615,7 +640,7 @@ public void testCreate_givenExistingModelSnapshot() { JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("foo"); - manager.create(jobTask, buildAutodetectParams(), e -> {}); + manager.create(jobTask, createJobDetails("foo"), buildAutodetectParams(), e -> {}); String expectedNotification = "Loading model snapshot [snapshot-1] with " + "latest_record_timestamp [1970-01-01T00:00:00.000Z], " + @@ -634,7 +659,7 @@ public void testCreate_givenNonZeroCountsAndNoModelSnapshotNorQuantiles() { JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("foo"); - manager.create(jobTask, buildAutodetectParams(), e -> {}); + manager.create(jobTask, createJobDetails("foo"), buildAutodetectParams(), e -> {}); String expectedNotification = "Loading model snapshot [N/A], " + "job latest_record_timestamp [1970-01-01T00:00:00.000Z]"; @@ -651,7 +676,13 @@ private AutodetectProcessManager createNonSpyManager(String jobId) { ExecutorService executorService = mock(ExecutorService.class); when(threadPool.executor(anyString())).thenReturn(executorService); when(threadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(mock(ThreadPool.Cancellable.class)); - when(jobManager.getJobOrThrowIfUnknown(jobId)).thenReturn(createJobDetails(jobId)); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(createJobDetails(jobId)); + return null; + }).when(jobManager).getJob(eq(jobId), any()); + AutodetectProcess autodetectProcess = mock(AutodetectProcess.class); AutodetectProcessFactory autodetectProcessFactory = (j, autodetectParams, e, onProcessCrash) -> autodetectProcess; @@ -685,7 +716,7 @@ private AutodetectProcessManager createManager(AutodetectCommunicator communicat autodetectProcessFactory, normalizerFactory, new NamedXContentRegistry(Collections.emptyList()), auditor); manager = spy(manager); - doReturn(communicator).when(manager).create(any(), eq(buildAutodetectParams()), any()); + doReturn(communicator).when(manager).create(any(), any(), eq(buildAutodetectParams()), any()); return manager; } From 92979f435236665985fa0787455e7763edf6b7ff Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 18 Sep 2018 12:48:55 +0100 Subject: [PATCH 05/57] [ML] Change Datafeed actions to read config from the config index (#33273) --- .../xpack/core/ml/MlMetadata.java | 3 +- .../core/ml/datafeed/DatafeedUpdate.java | 2 +- .../xpack/core/ml/job/messages/Messages.java | 1 + .../xpack/core/ml/utils/ExceptionsHelper.java | 4 + .../action/TransportDeleteDatafeedAction.java | 59 ++++--- .../action/TransportGetDatafeedsAction.java | 72 +++++++-- .../ml/action/TransportPutDatafeedAction.java | 93 +++++++---- .../action/TransportUpdateDatafeedAction.java | 95 +++++++---- .../persistence/DatafeedConfigProvider.java | 151 ++++++++++++++---- .../ml/job/persistence/JobConfigProvider.java | 21 +++ .../integration/DatafeedConfigProviderIT.java | 139 +++++++++++++--- .../ml/integration/JobConfigProviderIT.java | 38 +++++ .../xpack/ml/job/JobManagerTests.java | 59 +++---- .../ml/job/persistence/MockClientBuilder.java | 39 +++++ 14 files changed, 584 insertions(+), 192 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 14736a764390b..afb56dc6cc8c9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.core.ml; -import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; @@ -295,7 +294,7 @@ public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) { public Builder putDatafeed(DatafeedConfig datafeedConfig, Map headers) { if (datafeeds.containsKey(datafeedConfig.getId())) { - throw new ResourceAlreadyExistsException("A datafeed with id [" + datafeedConfig.getId() + "] already exists"); + throw ExceptionsHelper.datafeedAlreadyExists(datafeedConfig.getId()); } String jobId = datafeedConfig.getJobId(); checkJobIsAvailableForDatafeed(jobId); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java index d5425bdd1f469..6621f30179093 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java @@ -195,7 +195,7 @@ private void addOptionalField(XContentBuilder builder, ParseField field, Object } } - String getJobId() { + public String getJobId() { return jobId; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index b669e8f1edcfb..fff65ca2b4fae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -41,6 +41,7 @@ public final class Messages { public static final String DATAFEED_MISSING_MAX_AGGREGATION_FOR_TIME_FIELD = "Missing max aggregation for time_field [{0}]"; public static final String DATAFEED_FREQUENCY_MUST_BE_MULTIPLE_OF_AGGREGATIONS_INTERVAL = "Datafeed frequency [{0}] must be a multiple of the aggregation interval [{1}]"; + public static final String DATAFEED_ID_ALREADY_TAKEN = "A datafeed with id [{0}] already exists"; public static final String FILTER_NOT_FOUND = "No filter with id [{0}] exists"; public static final String FILTER_CONTAINS_TOO_MANY_ITEMS = "Filter [{0}] contains too many items; up to [{1}] items are allowed"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java index d5b83d25ce315..83bbe79a7b470 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java @@ -30,6 +30,10 @@ public static ResourceNotFoundException missingDatafeedException(String datafeed return new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); } + public static ResourceAlreadyExistsException datafeedAlreadyExists(String datafeedId) { + return new ResourceAlreadyExistsException(Messages.getMessage(Messages.DATAFEED_ID_ALREADY_TAKEN, datafeedId)); + } + public static ElasticsearchException serverError(String msg) { return new ElasticsearchException(msg); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java index 37210ce3c6ca2..f441e42acc517 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java @@ -11,42 +11,48 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class TransportDeleteDatafeedAction extends TransportMasterNodeAction { - private Client client; - private PersistentTasksService persistentTasksService; + private final Client client; + private final DatafeedConfigProvider datafeedConfigProvider; + private final ClusterService clusterService; + private final PersistentTasksService persistentTasksService; @Inject public TransportDeleteDatafeedAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Client client, PersistentTasksService persistentTasksService) { + Client client, PersistentTasksService persistentTasksService, + NamedXContentRegistry xContentRegistry) { super(settings, DeleteDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, DeleteDatafeedAction.Request::new); this.client = client; + this.datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry); this.persistentTasksService = persistentTasksService; + this.clusterService = clusterService; } @Override @@ -65,14 +71,14 @@ protected void masterOperation(DeleteDatafeedAction.Request request, ClusterStat if (request.isForce()) { forceDeleteDatafeed(request, state, listener); } else { - deleteDatafeedFromMetadata(request, listener); + deleteDatafeedConfig(request, listener); } } private void forceDeleteDatafeed(DeleteDatafeedAction.Request request, ClusterState state, ActionListener listener) { ActionListener finalListener = ActionListener.wrap( - response -> deleteDatafeedFromMetadata(request, listener), + response -> deleteDatafeedConfig(request, listener), listener::onFailure ); @@ -111,28 +117,19 @@ public void onFailure(Exception e) { } } - private void deleteDatafeedFromMetadata(DeleteDatafeedAction.Request request, ActionListener listener) { - clusterService.submitStateUpdateTask("delete-datafeed-" + request.getDatafeedId(), - new AckedClusterStateUpdateTask(request, listener) { - - @Override - protected AcknowledgedResponse newResponse(boolean acknowledged) { - return new AcknowledgedResponse(acknowledged); - } - - @Override - public ClusterState execute(ClusterState currentState) { - XPackPlugin.checkReadyForXPackCustomMetadata(currentState); - MlMetadata currentMetadata = MlMetadata.getMlMetadata(currentState); - PersistentTasksCustomMetaData persistentTasks = - currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) - .removeDatafeed(request.getDatafeedId(), persistentTasks).build(); - return ClusterState.builder(currentState).metaData( - MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()) - .build(); - } - }); + private void deleteDatafeedConfig(DeleteDatafeedAction.Request request, ActionListener listener) { + // Check datafeed is stopped + PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (MlTasks.getDatafeedTask(request.getDatafeedId(), tasks) != null) { + listener.onFailure(ExceptionsHelper.conflictStatusException( + Messages.getMessage(Messages.DATAFEED_CANNOT_DELETE_IN_CURRENT_STATE, request.getDatafeedId(), DatafeedState.STARTED))); + return; + } + + datafeedConfigProvider.deleteDatafeedConfig(request.getDatafeedId(), ActionListener.wrap( + deleteResponse -> listener.onResponse(new AcknowledgedResponse(true)), + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java index b4e3851eda820..1acf095102bdc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -15,25 +16,37 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; public class TransportGetDatafeedsAction extends TransportMasterNodeReadAction { + private final DatafeedConfigProvider datafeedConfigProvider; + @Inject - public TransportGetDatafeedsAction(Settings settings, TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, GetDatafeedsAction.NAME, transportService, clusterService, threadPool, actionFilters, - GetDatafeedsAction.Request::new, indexNameExpressionResolver); + public TransportGetDatafeedsAction(Settings settings, TransportService transportService, + ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, NamedXContentRegistry xContentRegistry) { + super(settings, GetDatafeedsAction.NAME, transportService, clusterService, threadPool, actionFilters, + GetDatafeedsAction.Request::new, indexNameExpressionResolver); + + datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry); } @Override @@ -48,18 +61,51 @@ protected GetDatafeedsAction.Response newResponse() { @Override protected void masterOperation(GetDatafeedsAction.Request request, ClusterState state, - ActionListener listener) throws Exception { + ActionListener listener) { logger.debug("Get datafeed '{}'", request.getDatafeedId()); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds()); - List datafeedConfigs = new ArrayList<>(); + Map clusterStateConfigs = + expandClusterStateDatafeeds(request.getDatafeedId(), request.allowNoDatafeeds(), state); + + datafeedConfigProvider.expandDatafeedConfigs(request.getDatafeedId(), request.allowNoDatafeeds(), ActionListener.wrap( + datafeedBuilders -> { + // Check for duplicate datafeeds + for (DatafeedConfig.Builder datafeed : datafeedBuilders) { + if (clusterStateConfigs.containsKey(datafeed.getId())) { + listener.onFailure(new IllegalStateException("Datafeed [" + datafeed.getId() + "] configuration " + + "exists in both clusterstate and index")); + return; + } + } + + // Merge cluster state and index configs + List datafeeds = new ArrayList<>(datafeedBuilders.size() + clusterStateConfigs.values().size()); + for (DatafeedConfig.Builder builder: datafeedBuilders) { + datafeeds.add(builder.build()); + } + + datafeeds.addAll(clusterStateConfigs.values()); + Collections.sort(datafeeds, Comparator.comparing(DatafeedConfig::getId)); + listener.onResponse(new GetDatafeedsAction.Response(new QueryPage<>(datafeeds, datafeeds.size(), + DatafeedConfig.RESULTS_FIELD))); + }, + listener::onFailure + )); + } + + Map expandClusterStateDatafeeds(String datafeedExpression, boolean allowNoDatafeeds, + ClusterState clusterState) { + + Map configById = new HashMap<>(); + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds); + for (String expandedDatafeedId : expandedDatafeedIds) { - datafeedConfigs.add(mlMetadata.getDatafeed(expandedDatafeedId)); + configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId)); } - listener.onResponse(new GetDatafeedsAction.Response(new QueryPage<>(datafeedConfigs, datafeedConfigs.size(), - DatafeedConfig.RESULTS_FIELD))); + return configById; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java index 60b8235ec84b7..dbde3d61d42b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java @@ -5,21 +5,23 @@ */ package org.elasticsearch.xpack.ml.action; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.license.LicenseUtils; @@ -28,16 +30,18 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.io.IOException; import java.util.Map; @@ -46,20 +50,26 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction headers, ActionListener listener) { - clusterService.submitStateUpdateTask( - "put-datafeed-" + request.getDatafeed().getId(), - new AckedClusterStateUpdateTask(request, listener) { + String datafeedId = request.getDatafeed().getId(); + String jobId = request.getDatafeed().getJobId(); + ElasticsearchException validationError = checkConfigsAreNotDefinedInClusterState(datafeedId, jobId); + if (validationError != null) { + listener.onFailure(validationError); + return; + } - @Override - protected PutDatafeedAction.Response newResponse(boolean acknowledged) { - if (acknowledged) { - logger.info("Created datafeed [{}]", request.getDatafeed().getId()); - } - return new PutDatafeedAction.Response(request.getDatafeed()); - } + CheckedConsumer validationOk = ok -> { + datafeedConfigProvider.putDatafeedConfig(request.getDatafeed(), headers, ActionListener.wrap( + indexResponse -> listener.onResponse(new PutDatafeedAction.Response(request.getDatafeed())), + listener::onFailure + )); + }; - @Override - public ClusterState execute(ClusterState currentState) { - return putDatafeed(request, headers, currentState); - } - }); + CheckedConsumer jobOk = ok -> + jobConfigProvider.validateDatafeedJob(request.getDatafeed(), ActionListener.wrap(validationOk, listener::onFailure)); + + checkJobDoesNotHaveADatafeed(jobId, ActionListener.wrap(jobOk, listener::onFailure)); + } + + /** + * Returns an exception if a datafeed with the same Id is defined in the + * cluster state or the job is in the cluster state and already has a datafeed + */ + @Nullable + private ElasticsearchException checkConfigsAreNotDefinedInClusterState(String datafeedId, String jobId) { + ClusterState clusterState = clusterService.state(); + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + + if (mlMetadata.getDatafeed(datafeedId) != null) { + return ExceptionsHelper.datafeedAlreadyExists(datafeedId); + } + + if (mlMetadata.getDatafeedByJobId(jobId).isPresent()) { + return ExceptionsHelper.conflictStatusException("Cannot create datafeed [" + datafeedId + "] as a " + + "job [" + jobId + "] defined in the cluster state references a datafeed with the same Id"); + } + + return null; } - private ClusterState putDatafeed(PutDatafeedAction.Request request, Map headers, ClusterState clusterState) { - XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); - MlMetadata currentMetadata = MlMetadata.getMlMetadata(clusterState); - MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) - .putDatafeed(request.getDatafeed(), headers).build(); - return ClusterState.builder(clusterState).metaData( - MetaData.builder(clusterState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()) - .build(); + private void checkJobDoesNotHaveADatafeed(String jobId, ActionListener listener) { + datafeedConfigProvider.findDatafeedForJobId(jobId, ActionListener.wrap( + datafeedIds -> { + if (datafeedIds.isEmpty()) { + listener.onResponse(Boolean.TRUE); + } else { + listener.onFailure(ExceptionsHelper.conflictStatusException("A datafeed [" + datafeedIds.iterator().next() + + "] already exists for job [" + jobId + "]")); + } + }, + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java index 8cf917c4405ea..6b17721b20d68 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java @@ -8,34 +8,45 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.util.Map; public class TransportUpdateDatafeedAction extends TransportMasterNodeAction { + private final DatafeedConfigProvider datafeedConfigProvider; + private final JobConfigProvider jobConfigProvider; + @Inject public TransportUpdateDatafeedAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, NamedXContentRegistry xContentRegistry) { super(settings, UpdateDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, UpdateDatafeedAction.Request::new); + + datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry); + jobConfigProvider = new JobConfigProvider(client, settings); } @Override @@ -50,34 +61,60 @@ protected PutDatafeedAction.Response newResponse() { @Override protected void masterOperation(UpdateDatafeedAction.Request request, ClusterState state, - ActionListener listener) { + ActionListener listener) throws Exception { final Map headers = threadPool.getThreadContext().getHeaders(); - clusterService.submitStateUpdateTask("update-datafeed-" + request.getUpdate().getId(), - new AckedClusterStateUpdateTask(request, listener) { - private volatile DatafeedConfig updatedDatafeed; + // Check datafeed is stopped + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (MlTasks.getDatafeedTask(request.getUpdate().getId(), tasks) != null) { + listener.onFailure(ExceptionsHelper.conflictStatusException( + Messages.getMessage(Messages.DATAFEED_CANNOT_UPDATE_IN_CURRENT_STATE, + request.getUpdate().getId(), DatafeedState.STARTED))); + return; + } - @Override - protected PutDatafeedAction.Response newResponse(boolean acknowledged) { - if (acknowledged) { - logger.info("Updated datafeed [{}]", request.getUpdate().getId()); - } - return new PutDatafeedAction.Response(updatedDatafeed); - } + String datafeedId = request.getUpdate().getId(); + + CheckedConsumer updateConsumer = ok -> { + datafeedConfigProvider.updateDatefeedConfig(request.getUpdate().getId(), request.getUpdate(), headers, + jobConfigProvider::validateDatafeedJob, + ActionListener.wrap( + updatedConfig -> listener.onResponse(new PutDatafeedAction.Response(updatedConfig)), + listener::onFailure + )); + }; + + + if (request.getUpdate().getJobId() != null) { + checkJobDoesNotHaveADifferentDatafeed(request.getUpdate().getJobId(), datafeedId, + ActionListener.wrap(updateConsumer, listener::onFailure)); + } else { + updateConsumer.accept(Boolean.TRUE); + } + } - @Override - public ClusterState execute(ClusterState currentState) { - DatafeedUpdate update = request.getUpdate(); - MlMetadata currentMetadata = MlMetadata.getMlMetadata(currentState); - PersistentTasksCustomMetaData persistentTasks = - currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) - .updateDatafeed(update, persistentTasks, headers).build(); - updatedDatafeed = newMetadata.getDatafeed(update.getId()); - return ClusterState.builder(currentState).metaData( - MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()).build(); + /* + * This is a check against changing the datafeed's jobId and that job + * already having a datafeed. + * The job the updated datafeed refers to should have no datafeed or + * if it does have a datafeed it must be the one we are updating + */ + private void checkJobDoesNotHaveADifferentDatafeed(String jobId, String datafeedId, ActionListener listener) { + datafeedConfigProvider.findDatafeedForJobId(jobId, ActionListener.wrap( + datafeedIds -> { + if (datafeedIds.isEmpty()) { + // Ok the job does not have a datafeed + listener.onResponse(Boolean.TRUE); + } else if (datafeedIds.size() == 1 && datafeedIds.contains(datafeedId)) { + // Ok the job has the datafeed being updated + listener.onResponse(Boolean.TRUE); + } else { + listener.onFailure(ExceptionsHelper.conflictStatusException("A datafeed [" + datafeedIds.iterator().next() + + "] already exists for job [" + jobId + "]")); } - }); + }, + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index 9702f1096ecf4..f0402ed869224 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; @@ -41,8 +42,10 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -57,6 +60,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -86,17 +91,40 @@ public DatafeedConfigProvider(Client client, Settings settings, NamedXContentReg * @param config The datafeed configuration * @param listener Index response listener */ - public void putDatafeedConfig(DatafeedConfig config, ActionListener listener) { + public void putDatafeedConfig(DatafeedConfig config, Map headers, ActionListener listener) { + + if (headers.isEmpty() == false) { + // Filter any values in headers that aren't security fields + DatafeedConfig.Builder builder = new DatafeedConfig.Builder(config); + Map securityHeaders = headers.entrySet().stream() + .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + builder.setHeaders(securityHeaders); + config = builder.build(); + } + + final String datafeedId = config.getId(); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder source = config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(config.getId())) + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)) .setSource(source) .setOpType(DocWriteRequest.OpType.CREATE) .request(); - executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + listener::onResponse, + e -> { + if (e instanceof VersionConflictEngineException) { + // the dafafeed already exists + listener.onFailure(ExceptionsHelper.datafeedAlreadyExists(datafeedId)); + } else { + listener.onFailure(e); + } + } + )); } catch (IOException e) { listener.onFailure(new ElasticsearchParseException("Failed to serialise datafeed config with id [" + config.getId() + "]", e)); @@ -131,6 +159,43 @@ public void onFailure(Exception e) { }); } + /** + * Find any datafeeds that are used by job {@code jobid} i.e. the + * datafeed that references job {@code jobid}. + * + * In theory there should never be more than one datafeed referencing a + * particular job. + * + * @param jobId The job to find + * @param listener Datafeed Id listener + */ + public void findDatafeedForJobId(String jobId, ActionListener> listener) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdQuery(jobId)); + sourceBuilder.fetchSource(false); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName()); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + Set datafeedIds = new HashSet<>(); + SearchHit[] hits = response.getHits().getHits(); + // There should be 0 or 1 datafeeds referencing the same job + assert hits.length <= 1; + + for (SearchHit hit : hits) { + datafeedIds.add(hit.field(DatafeedConfig.ID.getPreferredName()).getValue()); + } + + listener.onResponse(datafeedIds); + }, + listener::onFailure) + , client::search); + } + /** * Delete the datafeed config document * @@ -161,12 +226,19 @@ public void onFailure(Exception e) { * Get the datafeed config and apply the {@code update} * then index the modified config setting the version in the request. * + * The {@code validator} consumer can be used to perform extra validation + * but it must call the passed ActionListener. For example a no-op validator + * would be {@code (updatedConfig, listener) -> listener.onResponse(Boolean.TRUE)} + * * @param datafeedId The Id of the datafeed to update * @param update The update * @param headers Datafeed headers applied with the update + * @param validator BiConsumer that accepts the updated config and can perform + * extra validations. {@code validator} must call the passed listener * @param updatedConfigListener Updated datafeed config listener */ public void updateDatefeedConfig(String datafeedId, DatafeedUpdate update, Map headers, + BiConsumer> validator, ActionListener updatedConfigListener) { GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)); @@ -197,26 +269,19 @@ public void onResponse(GetResponse getResponse) { return; } - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - XContentBuilder updatedSource = updatedConfig.toXContent(builder, ToXContent.EMPTY_PARAMS); - IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(updatedConfig.getId())) - .setSource(updatedSource) - .setVersion(version) - .request(); - - executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( - indexResponse -> { - assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; - updatedConfigListener.onResponse(updatedConfig); - }, - updatedConfigListener::onFailure - )); + ActionListener validatedListener = ActionListener.wrap( + ok -> { + indexUpdatedConfig(updatedConfig, version, ActionListener.wrap( + indexResponse -> { + assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED; + updatedConfigListener.onResponse(updatedConfig); + }, + updatedConfigListener::onFailure)); + }, + updatedConfigListener::onFailure + ); - } catch (IOException e) { - updatedConfigListener.onFailure( - new ElasticsearchParseException("Failed to serialise datafeed config with id [" + datafeedId + "]", e)); - } + validator.accept(updatedConfig, validatedListener); } @Override @@ -226,6 +291,23 @@ public void onFailure(Exception e) { }); } + private void indexUpdatedConfig(DatafeedConfig updatedConfig, long version, ActionListener listener) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + XContentBuilder updatedSource = updatedConfig.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); + IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(updatedConfig.getId())) + .setSource(updatedSource) + .setVersion(version) + .request(); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + + } catch (IOException e) { + listener.onFailure( + new ElasticsearchParseException("Failed to serialise datafeed config with id [" + updatedConfig.getId() + "]", e)); + } + } + /** * Expands an expression into the set of matching names. {@code expresssion} * may be a wildcard, a datafeed ID or a list of those. @@ -252,10 +334,10 @@ public void onFailure(Exception e) { */ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); - String [] includes = new String[] {DatafeedConfig.ID.getPreferredName()}; - sourceBuilder.fetchSource(includes, null); + sourceBuilder.fetchSource(false); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName()); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -269,7 +351,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio Set datafeedIds = new HashSet<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { - datafeedIds.add((String)hit.getSourceAsMap().get(DatafeedConfig.ID.getPreferredName())); + datafeedIds.add(hit.field(DatafeedConfig.ID.getPreferredName()).getValue()); } requiredMatches.filterMatchedIds(datafeedIds); @@ -301,7 +383,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio // NORELEASE datafeed configs should be paged or have a mechanism to return all jobs if there are many of them public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) @@ -342,15 +424,15 @@ public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, A } - private QueryBuilder buildQuery(String [] tokens) { - QueryBuilder jobQuery = new TermQueryBuilder(DatafeedConfig.CONFIG_TYPE.getPreferredName(), DatafeedConfig.TYPE); + private QueryBuilder buildDatafeedIdQuery(String [] tokens) { + QueryBuilder datafeedQuery = new TermQueryBuilder(DatafeedConfig.CONFIG_TYPE.getPreferredName(), DatafeedConfig.TYPE); if (Strings.isAllOrWildcard(tokens)) { // match all - return jobQuery; + return datafeedQuery; } BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.filter(jobQuery); + boolQueryBuilder.filter(datafeedQuery); BoolQueryBuilder shouldQueries = new BoolQueryBuilder(); List terms = new ArrayList<>(); @@ -373,6 +455,13 @@ private QueryBuilder buildQuery(String [] tokens) { return boolQueryBuilder; } + private QueryBuilder buildDatafeedJobIdQuery(String jobId) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new TermQueryBuilder(DatafeedConfig.CONFIG_TYPE.getPreferredName(), DatafeedConfig.TYPE)); + boolQueryBuilder.filter(new TermQueryBuilder(Job.ID.getPreferredName(), jobId)); + return boolQueryBuilder; + } + private void parseLenientlyFromSource(BytesReference source, ActionListener datafeedConfigListener) { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index ae13d0371a3a5..ef593417a19e3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -46,6 +46,8 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; @@ -545,6 +547,25 @@ public void findJobsWithCustomRules(ActionListener> listener) { , client::search); } + /** + * Get the job reference by the datafeed and validate the datafeed config against it + * @param config Datafeed config + * @param listener Validation listener + */ + public void validateDatafeedJob(DatafeedConfig config, ActionListener listener) { + getJob(config.getJobId(), ActionListener.wrap( + jobBuilder -> { + try { + DatafeedJobValidator.validate(config, jobBuilder.build()); + listener.onResponse(Boolean.TRUE); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + )); + } + private void parseJobLenientlyFromSource(BytesReference source, ActionListener jobListener) { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index 8eeeb2908cf88..3f70baf4771c3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -5,12 +5,13 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -29,15 +30,18 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; public class DatafeedConfigProviderIT extends MlSingleNodeTestCase { - private DatafeedConfigProvider datafeedConfigProvider; @Before @@ -53,8 +57,8 @@ public void testCrud() throws InterruptedException { AtomicReference exceptionHolder = new AtomicReference<>(); // Create datafeed config - DatafeedConfig config = createDatafeedConfig(datafeedId, "j1"); - blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + DatafeedConfig.Builder config = createDatafeedConfig(datafeedId, "j1"); + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config.build(), createSecurityHeader(), actionListener), indexResponseHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertEquals(RestStatus.CREATED, indexResponseHolder.get().status()); @@ -64,7 +68,11 @@ public void testCrud() throws InterruptedException { blockingCall(actionListener -> datafeedConfigProvider.getDatafeedConfig(datafeedId, actionListener), configBuilderHolder, exceptionHolder); assertNull(exceptionHolder.get()); - assertEquals(config, configBuilderHolder.get().build()); + + // Headers are set by the putDatafeedConfig method so they + // must be added to the original config before equality testing + config.setHeaders(createSecurityHeader()); + assertEquals(config.build(), configBuilderHolder.get().build()); // Update DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedId); @@ -77,12 +85,21 @@ public void testCrud() throws InterruptedException { AtomicReference configHolder = new AtomicReference<>(); blockingCall(actionListener -> - datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), updateHeaders, actionListener), + datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), updateHeaders, + (updatedConfig, listener) -> listener.onResponse(Boolean.TRUE), actionListener), configHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertThat(configHolder.get().getIndices(), equalTo(updateIndices)); assertThat(configHolder.get().getHeaders().get(securityHeader), equalTo("CHANGED")); + // Read the updated config + configBuilderHolder.set(null); + blockingCall(actionListener -> datafeedConfigProvider.getDatafeedConfig(datafeedId, actionListener), + configBuilderHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(configBuilderHolder.get().build().getIndices(), equalTo(updateIndices)); + assertThat(configBuilderHolder.get().build().getHeaders().get(securityHeader), equalTo("CHANGED")); + // Delete AtomicReference deleteResponseHolder = new AtomicReference<>(); blockingCall(actionListener -> datafeedConfigProvider.deleteDatafeedConfig(datafeedId, actionListener), @@ -98,18 +115,19 @@ public void testMultipleCreateAndDeletes() throws InterruptedException { AtomicReference exceptionHolder = new AtomicReference<>(); // Create datafeed config - DatafeedConfig config = createDatafeedConfig(datafeedId, "j1"); - blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + DatafeedConfig.Builder config = createDatafeedConfig(datafeedId, "j1"); + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config.build(), Collections.emptyMap(), actionListener), indexResponseHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertEquals(RestStatus.CREATED, indexResponseHolder.get().status()); // cannot create another with the same id indexResponseHolder.set(null); - blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener), + blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config.build(), Collections.emptyMap(), actionListener), indexResponseHolder, exceptionHolder); assertNull(indexResponseHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(VersionConflictEngineException.class)); + assertThat(exceptionHolder.get(), instanceOf(ResourceAlreadyExistsException.class)); + assertEquals("A datafeed with Id [df2] already exists", exceptionHolder.get().getMessage()); // delete exceptionHolder.set(null); @@ -127,11 +145,11 @@ public void testMultipleCreateAndDeletes() throws InterruptedException { assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); } - public void testUpdateWithAValidationError() throws Exception { + public void testUpdateWhenApplyingTheUpdateThrows() throws Exception { final String datafeedId = "df-bad-update"; - DatafeedConfig config = createDatafeedConfig(datafeedId, "j2"); - putDatafeedConfig(config); + DatafeedConfig.Builder config = createDatafeedConfig(datafeedId, "j2"); + putDatafeedConfig(config, Collections.emptyMap()); DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedId); update.setId("wrong-datafeed-id"); @@ -139,7 +157,8 @@ public void testUpdateWithAValidationError() throws Exception { AtomicReference exceptionHolder = new AtomicReference<>(); AtomicReference configHolder = new AtomicReference<>(); blockingCall(actionListener -> - datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), Collections.emptyMap(), actionListener), + datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), Collections.emptyMap(), + (updatedConfig, listener) -> listener.onResponse(Boolean.TRUE), actionListener), configHolder, exceptionHolder); assertNull(configHolder.get()); assertNotNull(exceptionHolder.get()); @@ -147,6 +166,33 @@ public void testUpdateWithAValidationError() throws Exception { assertThat(exceptionHolder.get().getMessage(), containsString("Cannot apply update to datafeedConfig with different id")); } + public void testUpdateWithValidatorFunctionThatErrors() throws Exception { + final String datafeedId = "df-validated-update"; + + DatafeedConfig.Builder config = createDatafeedConfig(datafeedId, "hob-job"); + putDatafeedConfig(config, Collections.emptyMap()); + + DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedId); + List updateIndices = Collections.singletonList("a-different-index"); + update.setIndices(updateIndices); + + BiConsumer> validateErrorFunction = (updatedConfig, listener) -> { + new Thread(() -> listener.onFailure(new IllegalArgumentException("this is a bad update")), getTestName()).start(); + }; + + AtomicReference configHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + blockingCall(actionListener -> + datafeedConfigProvider.updateDatefeedConfig(datafeedId, update.build(), Collections.emptyMap(), + validateErrorFunction, actionListener), + configHolder, exceptionHolder); + + assertNull(configHolder.get()); + assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(IllegalArgumentException.class)); + assertThat(exceptionHolder.get().getMessage(), containsString("this is a bad update")); + + } + public void testAllowNoDatafeeds() throws InterruptedException { AtomicReference> datafeedIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); @@ -182,11 +228,11 @@ public void testAllowNoDatafeeds() throws InterruptedException { } public void testExpandDatafeeds() throws Exception { - DatafeedConfig foo1 = putDatafeedConfig(createDatafeedConfig("foo-1", "j1")); - DatafeedConfig foo2 = putDatafeedConfig(createDatafeedConfig("foo-2", "j2")); - DatafeedConfig bar1 = putDatafeedConfig(createDatafeedConfig("bar-1", "j3")); - DatafeedConfig bar2 = putDatafeedConfig(createDatafeedConfig("bar-2", "j4")); - putDatafeedConfig(createDatafeedConfig("not-used", "j5")); + DatafeedConfig foo1 = putDatafeedConfig(createDatafeedConfig("foo-1", "j1"), Collections.emptyMap()); + DatafeedConfig foo2 = putDatafeedConfig(createDatafeedConfig("foo-2", "j2"), Collections.emptyMap()); + DatafeedConfig bar1 = putDatafeedConfig(createDatafeedConfig("bar-1", "j3"), Collections.emptyMap()); + DatafeedConfig bar2 = putDatafeedConfig(createDatafeedConfig("bar-2", "j4"), Collections.emptyMap()); + putDatafeedConfig(createDatafeedConfig("not-used", "j5"), Collections.emptyMap()); client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); @@ -234,20 +280,65 @@ public void testExpandDatafeeds() throws Exception { assertThat(expandedDatafeeds, containsInAnyOrder(bar1, foo1, foo2)); } - private DatafeedConfig createDatafeedConfig(String id, String jobId) { + public void testFindDatafeedForJobId() throws Exception { + putDatafeedConfig(createDatafeedConfig("foo-1", "j1"), Collections.emptyMap()); + putDatafeedConfig(createDatafeedConfig("foo-2", "j2"), Collections.emptyMap()); + putDatafeedConfig(createDatafeedConfig("bar-1", "j3"), Collections.emptyMap()); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + AtomicReference> datafeedIdsHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("new-job", actionListener), + datafeedIdsHolder, exceptionHolder); + assertThat(datafeedIdsHolder.get(), empty()); + + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("j2", actionListener), + datafeedIdsHolder, exceptionHolder); + assertThat(datafeedIdsHolder.get(), contains("foo-2")); + + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("j3", actionListener), + datafeedIdsHolder, exceptionHolder); + assertThat(datafeedIdsHolder.get(), contains("bar-1")); + } + + public void testHeadersAreOverwritten() throws Exception { + String dfId = "df-with-headers"; + DatafeedConfig.Builder configWithUnrelatedHeaders = createDatafeedConfig(dfId, "j1"); + Map headers = new HashMap<>(); + headers.put("UNRELATED-FIELD", "WILL-BE-FILTERED"); + configWithUnrelatedHeaders.setHeaders(headers); + + putDatafeedConfig(configWithUnrelatedHeaders, createSecurityHeader()); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference configBuilderHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.getDatafeedConfig(dfId, actionListener), + configBuilderHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(configBuilderHolder.get().build().getHeaders().entrySet(), hasSize(1)); + assertEquals(configBuilderHolder.get().build().getHeaders(), createSecurityHeader()); + } + + private DatafeedConfig.Builder createDatafeedConfig(String id, String jobId) { DatafeedConfig.Builder builder = new DatafeedConfig.Builder(id, jobId); builder.setIndices(Collections.singletonList("beats*")); + return builder; + } + private Map createSecurityHeader() { Map headers = new HashMap<>(); // Only security headers are updated, grab the first one String securityHeader = ClientHelper.SECURITY_HEADER_FILTERS.iterator().next(); headers.put(securityHeader, "SECURITY_"); - builder.setHeaders(headers); - return builder.build(); + return headers; } - private DatafeedConfig putDatafeedConfig(DatafeedConfig config) throws Exception { - this.blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, actionListener)); + private DatafeedConfig putDatafeedConfig(DatafeedConfig.Builder builder, Map headers) throws Exception { + builder.setHeaders(headers); + DatafeedConfig config = builder.build(); + this.blockingCall(actionListener -> datafeedConfigProvider.putDatafeedConfig(config, headers, actionListener)); return config; } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index 712206e37556a..3456290745a05 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -13,6 +13,11 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; @@ -22,6 +27,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.config.RuleScope; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; @@ -419,6 +425,38 @@ public void testFindJobsWithCustomRules() throws Exception { assertThat(foundJobIds, containsInAnyOrder(jobWithRules1.getId(), jobWithRules2.getId())); } + public void testValidateDatafeedJob() throws Exception { + String jobId = "validate-df-job"; + putJob(createJob(jobId, Collections.emptyList())); + + AtomicReference responseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + DatafeedConfig.Builder builder = new DatafeedConfig.Builder("df1", jobId); + builder.setIndices(Collections.singletonList("data-index")); + DatafeedConfig config = builder.build(); + + blockingCall(listener -> jobConfigProvider.validateDatafeedJob(config, listener), responseHolder, exceptionHolder); + assertTrue(responseHolder.get()); + assertNull(exceptionHolder.get()); + + builder = new DatafeedConfig.Builder("df1", jobId); + builder.setIndices(Collections.singletonList("data-index")); + + // This config is not valid because it uses aggs but the job's + // summary count field is not set + MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time"); + HistogramAggregationBuilder histogram = + AggregationBuilders.histogram("time").interval(1800.0).field("time").subAggregation(maxTime); + builder.setAggregations(new AggregatorFactories.Builder().addAggregator(histogram)); + DatafeedConfig badConfig = builder.build(); + + blockingCall(listener -> jobConfigProvider.validateDatafeedJob(badConfig, listener), responseHolder, exceptionHolder); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ElasticsearchStatusException.class)); + assertEquals(Messages.DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD, exceptionHolder.get().getMessage()); + } + private static Job.Builder createJob(String jobId, List groups) { Detector.Builder d1 = new Detector.Builder("info_content", "domain"); d1.setOverFieldName("client"); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index 98c406694ec5c..c593444297040 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -57,6 +58,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; @@ -442,20 +444,10 @@ public void testNotifyFilterChangedGivenOnlyRemovedItems() throws IOException { } public void testUpdateProcessOnCalendarChanged() throws IOException { - List docsAsBytes = new ArrayList<>(); - Job.Builder job1 = buildJobBuilder("job-1"); - docsAsBytes.add(toBytesReference(job1.build())); - Job.Builder job2 = buildJobBuilder("job-2"); -// docsAsBytes.add(toBytesReference(job2.build())); - Job.Builder job3 = buildJobBuilder("job-3"); - docsAsBytes.add(toBytesReference(job3.build())); - Job.Builder job4 = buildJobBuilder("job-4"); - docsAsBytes.add(toBytesReference(job4.build())); - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask(job1.getId(), "node_id", JobState.OPENED, tasksBuilder); - addJobTask(job2.getId(), "node_id", JobState.OPENED, tasksBuilder); - addJobTask(job3.getId(), "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-1", "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-2", "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-3", "node_id", JobState.OPENED, tasksBuilder); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() @@ -464,7 +456,10 @@ public void testUpdateProcessOnCalendarChanged() throws IOException { when(clusterService.state()).thenReturn(clusterState); MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); - mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + // For the JobConfigProvider expand groups search. + // The search will not return any results + mockClientBuilder.prepareSearchFields(AnomalyDetectorsIndex.configIndexName(), Collections.emptyList()); + JobManager jobManager = createJobManager(mockClientBuilder.build()); jobManager.updateProcessOnCalendarChanged(Arrays.asList("job-1", "job-3", "job-4"), @@ -478,28 +473,17 @@ public void testUpdateProcessOnCalendarChanged() throws IOException { List capturedUpdateParams = updateParamsCaptor.getAllValues(); assertThat(capturedUpdateParams.size(), equalTo(2)); - assertThat(capturedUpdateParams.get(0).getJobId(), equalTo(job1.getId())); + assertThat(capturedUpdateParams.get(0).getJobId(), equalTo("job-1")); assertThat(capturedUpdateParams.get(0).isUpdateScheduledEvents(), is(true)); - assertThat(capturedUpdateParams.get(1).getJobId(), equalTo(job3.getId())); + assertThat(capturedUpdateParams.get(1).getJobId(), equalTo("job-3")); assertThat(capturedUpdateParams.get(1).isUpdateScheduledEvents(), is(true)); } public void testUpdateProcessOnCalendarChanged_GivenGroups() throws IOException { - Job.Builder job1 = buildJobBuilder("job-1"); - job1.setGroups(Collections.singletonList("group-1")); - Job.Builder job2 = buildJobBuilder("job-2"); - job2.setGroups(Collections.singletonList("group-1")); - Job.Builder job3 = buildJobBuilder("job-3"); - - List docsAsBytes = new ArrayList<>(); - docsAsBytes.add(toBytesReference(job1.build())); - docsAsBytes.add(toBytesReference(job2.build())); -// docsAsBytes.add(toBytesReference(job3.build())); - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask(job1.getId(), "node_id", JobState.OPENED, tasksBuilder); - addJobTask(job2.getId(), "node_id", JobState.OPENED, tasksBuilder); - addJobTask(job3.getId(), "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-1", "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-2", "node_id", JobState.OPENED, tasksBuilder); + addJobTask("job-3", "node_id", JobState.OPENED, tasksBuilder); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder() @@ -508,7 +492,16 @@ public void testUpdateProcessOnCalendarChanged_GivenGroups() throws IOException when(clusterService.state()).thenReturn(clusterState); MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test"); - mockClientBuilder.prepareSearch(AnomalyDetectorsIndex.configIndexName(), docsAsBytes); + // For the JobConfigProvider expand groups search. + // group-1 will expand to job-1 and job-2 + List> fieldHits = new ArrayList<>(); + fieldHits.add(Collections.singletonMap(Job.ID.getPreferredName(), + new DocumentField(Job.ID.getPreferredName(), Collections.singletonList("job-1")))); + fieldHits.add(Collections.singletonMap(Job.ID.getPreferredName(), + new DocumentField(Job.ID.getPreferredName(), Collections.singletonList("job-2")))); + + + mockClientBuilder.prepareSearchFields(AnomalyDetectorsIndex.configIndexName(), fieldHits); JobManager jobManager = createJobManager(mockClientBuilder.build()); jobManager.updateProcessOnCalendarChanged(Collections.singletonList("group-1"), @@ -522,9 +515,9 @@ public void testUpdateProcessOnCalendarChanged_GivenGroups() throws IOException List capturedUpdateParams = updateParamsCaptor.getAllValues(); assertThat(capturedUpdateParams.size(), equalTo(2)); - assertThat(capturedUpdateParams.get(0).getJobId(), equalTo(job1.getId())); + assertThat(capturedUpdateParams.get(0).getJobId(), equalTo("job-1")); assertThat(capturedUpdateParams.get(0).isUpdateScheduledEvents(), is(true)); - assertThat(capturedUpdateParams.get(1).getJobId(), equalTo(job2.getId())); + assertThat(capturedUpdateParams.get(1).getJobId(), equalTo("job-2")); assertThat(capturedUpdateParams.get(1).isUpdateScheduledEvents(), is(true)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index a5f3d5ff5179c..726b815728f52 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -41,6 +41,7 @@ import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -57,6 +58,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertArrayEquals; @@ -308,6 +310,43 @@ public Void answer(InvocationOnMock invocationOnMock) { return this; } + /* + * Mock a search that returns search hits with fields. + * The number of hits is the size of fields + */ + @SuppressWarnings("unchecked") + public MockClientBuilder prepareSearchFields(String indexName, List> fields) { + SearchRequestBuilder builder = mock(SearchRequestBuilder.class); + when(builder.setIndicesOptions(any())).thenReturn(builder); + when(builder.setQuery(any())).thenReturn(builder); + when(builder.setSource(any())).thenReturn(builder); + SearchRequest request = new SearchRequest(indexName); + when(builder.request()).thenReturn(request); + + when(client.prepareSearch(eq(indexName))).thenReturn(builder); + + SearchHit hits [] = new SearchHit[fields.size()]; + for (int i=0; i() { + @Override + public Void answer(InvocationOnMock invocationOnMock) { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(response); + return null; + } + }).when(client).search(eq(request), any()); + + return this; + } + public MockClientBuilder prepareSearchAnySize(String index, String type, SearchResponse response, ArgumentCaptor filter) { SearchRequestBuilder builder = mock(SearchRequestBuilder.class); when(builder.setTypes(eq(type))).thenReturn(builder); From b34eef1bbccac6bea410c466874a62ba32b59c79 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 27 Sep 2018 10:10:30 +0100 Subject: [PATCH 06/57] [ML] Allocate jobs based on JobParams rather than cluster state config (#33994) --- .../xpack/core/XPackClientPlugin.java | 9 +- .../elasticsearch/xpack/core/ml/MlTasks.java | 51 +++- .../xpack/core/ml/action/OpenJobAction.java | 31 ++- .../core/ml/action/StartDatafeedAction.java | 7 +- .../xpack/core/ml/datafeed/DatafeedState.java | 4 +- .../core/ml/job/config/JobTaskState.java | 4 +- .../xpack/core/ml/job/config/JobUpdate.java | 41 ++- .../xpack/core}/ml/MlTasksTests.java | 30 ++- .../core/ml/job/config/JobUpdateTests.java | 3 + .../MlNativeAutodetectIntegTestCase.java | 5 +- .../xpack/ml/MachineLearning.java | 3 + .../xpack/ml/MlAssignmentNotifier.java | 5 +- .../ml/action/TransportOpenJobAction.java | 230 +++++++--------- .../action/TransportStartDatafeedAction.java | 4 +- .../ml/job/persistence/JobConfigProvider.java | 48 ++++ .../xpack/ml/MlMetadataTests.java | 4 +- .../action/TransportCloseJobActionTests.java | 2 +- .../action/TransportOpenJobActionTests.java | 248 +++++------------- .../TransportStopDatafeedActionTests.java | 4 +- .../integration/BasicDistributedJobsIT.java | 2 +- .../ml/integration/JobConfigProviderIT.java | 19 +- 21 files changed, 383 insertions(+), 371 deletions(-) rename x-pack/plugin/{ml/src/test/java/org/elasticsearch/xpack => core/src/test/java/org/elasticsearch/xpack/core}/ml/MlTasksTests.java (71%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 21bd005ac5b7c..f142caaa52df9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction; @@ -334,9 +335,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new), // ML - Persistent action requests - new NamedWriteableRegistry.Entry(PersistentTaskParams.class, StartDatafeedAction.TASK_NAME, + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME, StartDatafeedAction.DatafeedParams::new), - new NamedWriteableRegistry.Entry(PersistentTaskParams.class, OpenJobAction.TASK_NAME, + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME, OpenJobAction.JobParams::new), // ML - Task states new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new), @@ -384,9 +385,9 @@ public List getNamedXContent() { new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"), parser -> MlMetadata.LENIENT_PARSER.parse(parser, null).build()), // ML - Persistent action requests - new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(StartDatafeedAction.TASK_NAME), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.DATAFEED_TASK_NAME), StartDatafeedAction.DatafeedParams::fromXContent), - new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(OpenJobAction.TASK_NAME), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.JOB_TASK_NAME), OpenJobAction.JobParams::fromXContent), // ML - Task states new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(DatafeedState.NAME), DatafeedState::fromXContent), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index f421ba7bf4ad8..a56d3d639239d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -12,14 +12,17 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; -import java.util.Collection; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; public final class MlTasks { - public static final String JOB_TASK_PREFIX = "job-"; - public static final String DATAFEED_TASK_PREFIX = "datafeed-"; + public static final String JOB_TASK_NAME = "xpack/ml/job"; + public static final String DATAFEED_TASK_NAME = "xpack/ml/datafeed"; + + private static final String JOB_TASK_ID_PREFIX = "job-"; + private static final String DATAFEED_TASK_ID_PREFIX = "datafeed-"; private MlTasks() { } @@ -29,7 +32,7 @@ private MlTasks() { * A datafeed id can be used as a job id, because they are stored separately in cluster state. */ public static String jobTaskId(String jobId) { - return JOB_TASK_PREFIX + jobId; + return JOB_TASK_ID_PREFIX + jobId; } /** @@ -37,7 +40,7 @@ public static String jobTaskId(String jobId) { * A job id can be used as a datafeed id, because they are stored separately in cluster state. */ public static String datafeedTaskId(String datafeedId) { - return DATAFEED_TASK_PREFIX + datafeedId; + return DATAFEED_TASK_ID_PREFIX + datafeedId; } @Nullable @@ -76,15 +79,41 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis } /** - * The job Ids of anomaly detector job tasks - * @param tasks Active tasks + * The job Ids of anomaly detector job tasks. + * All anomaly detector jobs are returned regardless of the status of the + * task (OPEN, CLOSED, FAILED etc). + * + * @param tasks Persistent tasks * @return The job Ids of anomaly detector job tasks */ public static Set openJobIds(PersistentTasksCustomMetaData tasks) { - Collection> activeTasks = tasks.tasks(); - - return activeTasks.stream().filter(t -> t.getId().startsWith(JOB_TASK_PREFIX)) - .map(t -> t.getId().substring(JOB_TASK_PREFIX.length())) + return tasks.findTasks(JOB_TASK_NAME, task -> true) + .stream() + .map(t -> t.getId().substring(JOB_TASK_ID_PREFIX.length())) .collect(Collectors.toSet()); } + + /** + * Is there an ml anomaly detector job task for the job {@code jobId}? + * @param jobId The job id + * @param tasks Persistent tasks + * @return + */ + public static boolean taskExistsForJob(String jobId, PersistentTasksCustomMetaData tasks) { + return openJobIds(tasks).contains(jobId); + } + + /** + * Read the active anomaly detector job tasks. + * Active tasks are not {@code JobState.CLOSED} or {@code JobState.FAILED}. + * + * @param tasks Persistent tasks + * @return The job tasks excluding closed and failed jobs + */ + public static List> activeJobTasks(PersistentTasksCustomMetaData tasks) { + return tasks.findTasks(JOB_TASK_NAME, task -> true) + .stream() + .filter(task -> ((JobTaskState) task.getState()).getState().isAnyOf(JobState.CLOSED, JobState.FAILED) == false) + .collect(Collectors.toList()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index 718a109b03d00..738eeb7f28335 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -26,6 +27,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.MachineLearningField; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -36,7 +38,7 @@ public class OpenJobAction extends Action { public static final OpenJobAction INSTANCE = new OpenJobAction(); public static final String NAME = "cluster:admin/xpack/ml/job/open"; - public static final String TASK_NAME = "xpack/ml/job"; + private OpenJobAction() { super(NAME); @@ -132,10 +134,9 @@ public static class JobParams implements XPackPlugin.XPackPersistentTaskParams { /** TODO Remove in 7.0.0 */ public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); - public static final ParseField TIMEOUT = new ParseField("timeout"); - public static ObjectParser PARSER = new ObjectParser<>(TASK_NAME, true, JobParams::new); + public static ObjectParser PARSER = new ObjectParser<>(MlTasks.JOB_TASK_NAME, true, JobParams::new); static { PARSER.declareString(JobParams::setJobId, Job.ID); PARSER.declareBoolean((p, v) -> {}, IGNORE_DOWNTIME); @@ -159,6 +160,7 @@ public static JobParams parseRequest(String jobId, XContentParser parser) { // A big state can take a while to restore. For symmetry with the _close endpoint any // changes here should be reflected there too. private TimeValue timeout = MachineLearningField.STATE_PERSIST_RESTORE_TIMEOUT; + private Job job; JobParams() { } @@ -170,6 +172,9 @@ public JobParams(String jobId) { public JobParams(StreamInput in) throws IOException { jobId = in.readString(); timeout = TimeValue.timeValueMillis(in.readVLong()); + if (in.getVersion().onOrAfter(Version.CURRENT)) { + job = in.readOptionalWriteable(Job::new); + } } public String getJobId() { @@ -188,15 +193,27 @@ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } + @Nullable + public Job getJob() { + return job; + } + + public void setJob(Job job) { + this.job = job; + } + @Override public String getWriteableName() { - return TASK_NAME; + return MlTasks.JOB_TASK_NAME; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); out.writeVLong(timeout.millis()); + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalWriteable(job); + } } @Override @@ -205,12 +222,13 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(Job.ID.getPreferredName(), jobId); builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep()); builder.endObject(); + // The job field is streamed but not persisted return builder; } @Override public int hashCode() { - return Objects.hash(jobId, timeout); + return Objects.hash(jobId, timeout, job); } @Override @@ -223,7 +241,8 @@ public boolean equals(Object obj) { } OpenJobAction.JobParams other = (OpenJobAction.JobParams) obj; return Objects.equals(jobId, other.jobId) && - Objects.equals(timeout, other.timeout); + Objects.equals(timeout, other.timeout) && + Objects.equals(job, other.job); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index 57a602b4cf09d..a8841578f28c7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -42,7 +43,6 @@ public class StartDatafeedAction extends Action { public static final StartDatafeedAction INSTANCE = new StartDatafeedAction(); public static final String NAME = "cluster:admin/xpack/ml/datafeed/start"; - public static final String TASK_NAME = "xpack/ml/datafeed"; private StartDatafeedAction() { super(NAME); @@ -141,8 +141,7 @@ public boolean equals(Object obj) { public static class DatafeedParams implements XPackPlugin.XPackPersistentTaskParams { - public static ObjectParser PARSER = new ObjectParser<>(TASK_NAME, true, DatafeedParams::new); - + public static ObjectParser PARSER = new ObjectParser<>(MlTasks.DATAFEED_TASK_NAME, true, DatafeedParams::new); static { PARSER.declareString((params, datafeedId) -> params.datafeedId = datafeedId, DatafeedConfig.ID); PARSER.declareString((params, startTime) -> params.startTime = parseDateOrThrow( @@ -229,7 +228,7 @@ public void setTimeout(TimeValue timeout) { @Override public String getWriteableName() { - return TASK_NAME; + return MlTasks.DATAFEED_TASK_NAME; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedState.java index 70102f27a5669..2fc2b5f9f45a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedState.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.persistent.PersistentTaskState; -import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.core.ml.MlTasks; import java.io.IOException; import java.util.Locale; @@ -23,7 +23,7 @@ public enum DatafeedState implements PersistentTaskState { STARTED, STOPPED, STARTING, STOPPING; - public static final String NAME = StartDatafeedAction.TASK_NAME; + public static final String NAME = MlTasks.DATAFEED_TASK_NAME; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, args -> fromString((String) args[0])); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index d9ab3357319c6..2e6cc4b99c4bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.MlTasks; import java.io.IOException; import java.util.Objects; @@ -23,7 +23,7 @@ public class JobTaskState implements PersistentTaskState { - public static final String NAME = OpenJobAction.TASK_NAME; + public static final String NAME = MlTasks.JOB_TASK_NAME; private static ParseField STATE = new ParseField("state"); private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index cdfd9bad7f1de..5e947d4fe8c28 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -29,6 +29,7 @@ public class JobUpdate implements Writeable, ToXContentObject { public static final ParseField DETECTORS = new ParseField("detectors"); + public static final ParseField CLEAR_JOB_FINISH_TIME = new ParseField("clear_job_finish_time"); // For internal updates static final ConstructingObjectParser INTERNAL_PARSER = new ConstructingObjectParser<>( @@ -59,6 +60,7 @@ public class JobUpdate implements Writeable, ToXContentObject { INTERNAL_PARSER.declareLong(Builder::setEstablishedModelMemory, Job.ESTABLISHED_MODEL_MEMORY); INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION); + INTERNAL_PARSER.declareBoolean(Builder::setClearJobFinishTime, CLEAR_JOB_FINISH_TIME); } private final String jobId; @@ -77,6 +79,7 @@ public class JobUpdate implements Writeable, ToXContentObject { private final Version modelSnapshotMinVersion; private final Long establishedModelMemory; private final Version jobVersion; + private final Boolean clearJobFinishTime; private JobUpdate(String jobId, @Nullable List groups, @Nullable String description, @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @@ -85,7 +88,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable Long modelSnapshotRetentionDays, @Nullable List categorisationFilters, @Nullable Map customSettings, @Nullable String modelSnapshotId, @Nullable Version modelSnapshotMinVersion, @Nullable Long establishedModelMemory, - @Nullable Version jobVersion) { + @Nullable Version jobVersion, @Nullable Boolean clearJobFinishTime) { this.jobId = jobId; this.groups = groups; this.description = description; @@ -102,6 +105,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotMinVersion = modelSnapshotMinVersion; this.establishedModelMemory = establishedModelMemory; this.jobVersion = jobVersion; + this.clearJobFinishTime = clearJobFinishTime; } public JobUpdate(StreamInput in) throws IOException { @@ -146,6 +150,11 @@ public JobUpdate(StreamInput in) throws IOException { } else { modelSnapshotMinVersion = null; } + if (in.getVersion().onOrAfter(Version.CURRENT)) { // NORELEASE change current to Jindex release version + clearJobFinishTime = in.readOptionalBoolean(); + } else { + clearJobFinishTime = null; + } } @Override @@ -191,6 +200,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } + if (out.getVersion().onOrAfter(Version.CURRENT)) { // NORELEASE change current to Jindex release version + out.writeOptionalBoolean(clearJobFinishTime); + } } public String getJobId() { @@ -257,6 +269,10 @@ public Version getJobVersion() { return jobVersion; } + public Boolean getClearJobFinishTime() { + return clearJobFinishTime; + } + public boolean isAutodetectProcessUpdate() { return modelPlotConfig != null || detectorUpdates != null || groups != null; } @@ -310,6 +326,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (jobVersion != null) { builder.field(Job.JOB_VERSION.getPreferredName(), jobVersion); } + if (clearJobFinishTime != null) { + builder.field(CLEAR_JOB_FINISH_TIME.getPreferredName(), clearJobFinishTime); + } builder.endObject(); return builder; } @@ -445,6 +464,10 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { builder.setJobVersion(jobVersion); } + if (clearJobFinishTime != null && clearJobFinishTime) { + builder.setFinishedTime(null); + } + builder.setAnalysisConfig(newAnalysisConfig); return builder.build(); } @@ -465,7 +488,8 @@ && updatesDetectors(job) == false && (modelSnapshotId == null || Objects.equals(modelSnapshotId, job.getModelSnapshotId())) && (modelSnapshotMinVersion == null || Objects.equals(modelSnapshotMinVersion, job.getModelSnapshotMinVersion())) && (establishedModelMemory == null || Objects.equals(establishedModelMemory, job.getEstablishedModelMemory())) - && (jobVersion == null || Objects.equals(jobVersion, job.getJobVersion())); + && (jobVersion == null || Objects.equals(jobVersion, job.getJobVersion())) + && (clearJobFinishTime == false || job.getFinishedTime() == null); } boolean updatesDetectors(Job job) { @@ -513,14 +537,15 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) - && Objects.equals(this.jobVersion, that.jobVersion); + && Objects.equals(this.jobVersion, that.jobVersion) + && Objects.equals(this.clearJobFinishTime, that.clearJobFinishTime); } @Override public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion); + modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion, clearJobFinishTime); } public static class DetectorUpdate implements Writeable, ToXContentObject { @@ -632,6 +657,7 @@ public static class Builder { private Version modelSnapshotMinVersion; private Long establishedModelMemory; private Version jobVersion; + private Boolean clearJobFinishTime; public Builder(String jobId) { this.jobId = jobId; @@ -727,10 +753,15 @@ public Builder setJobVersion(String version) { return this; } + public Builder setClearJobFinishTime(boolean clearJobFinishTime) { + this.clearJobFinishTime = clearJobFinishTime; + return this; + } + public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion); + modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion, clearJobFinishTime); } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java similarity index 71% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index 53bdfbdcb3b69..e2db7c3a30951 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -4,11 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml; +package org.elasticsearch.xpack.core.ml; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; @@ -24,7 +23,7 @@ public void testGetJobState() { // A missing task is a closed job assertEquals(JobState.CLOSED, MlTasks.getJobState("foo", tasksBuilder.build())); // A task with no status is opening - tasksBuilder.addTask(MlTasks.jobTaskId("foo"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo"), + tasksBuilder.addTask(MlTasks.jobTaskId("foo"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo"), new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); assertEquals(JobState.OPENING, MlTasks.getJobState("foo", tasksBuilder.build())); @@ -37,7 +36,7 @@ public void testGetDatefeedState() { // A missing task is a stopped datafeed assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build())); - tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME, + tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo", 0L), new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build())); @@ -50,7 +49,7 @@ public void testGetJobTask() { assertNull(MlTasks.getJobTask("foo", null)); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.jobTaskId("foo"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo"), + tasksBuilder.addTask(MlTasks.jobTaskId("foo"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo"), new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); assertNotNull(MlTasks.getJobTask("foo", tasksBuilder.build())); @@ -61,7 +60,7 @@ public void testGetDatafeedTask() { assertNull(MlTasks.getDatafeedTask("foo", null)); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME, + tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo", 0L), new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); @@ -73,14 +72,27 @@ public void testOpenJobIds() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); assertThat(MlTasks.openJobIds(tasksBuilder.build()), empty()); - tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo-1"), + tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); - tasksBuilder.addTask(MlTasks.jobTaskId("bar"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("bar"), + tasksBuilder.addTask(MlTasks.jobTaskId("bar"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("bar"), new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); - tasksBuilder.addTask(MlTasks.datafeedTaskId("df"), StartDatafeedAction.TASK_NAME, + tasksBuilder.addTask(MlTasks.datafeedTaskId("df"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df", 0L), new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); assertThat(MlTasks.openJobIds(tasksBuilder.build()), containsInAnyOrder("foo-1", "bar")); } + + public void testTaskExistsForJob() { + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + assertFalse(MlTasks.taskExistsForJob("job-1", tasksBuilder.build())); + + tasksBuilder.addTask(MlTasks.jobTaskId("foo"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.jobTaskId("bar"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("bar"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + + assertFalse(MlTasks.taskExistsForJob("job-1", tasksBuilder.build())); + assertTrue(MlTasks.taskExistsForJob("foo", tasksBuilder.build())); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 9aedf61859d32..35f72ced1e13b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -96,6 +96,9 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (useInternalParser && randomBoolean()) { update.setJobVersion(randomFrom(Version.CURRENT, Version.V_6_2_0, Version.V_6_1_0)); } + if (useInternalParser) { + update.setClearJobFinishTime(randomBoolean()); + } return update.build(); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java index 1fd0eddf41ced..6a621ffb076f0 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; @@ -445,9 +446,9 @@ protected void ensureClusterStateConsistency() throws IOException { List entries = new ArrayList<>(ClusterModule.getNamedWriteables()); entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables()); entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, StartDatafeedAction.TASK_NAME, + entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME, StartDatafeedAction.DatafeedParams::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, OpenJobAction.TASK_NAME, + entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME, OpenJobAction.JobParams::new)); entries.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new)); entries.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 0e22c20e43416..d316ad67b0692 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -165,6 +165,7 @@ import org.elasticsearch.xpack.ml.job.UpdateJobProcessNotifier; import org.elasticsearch.xpack.ml.job.categorization.MlClassicTokenizer; import org.elasticsearch.xpack.ml.job.categorization.MlClassicTokenizerFactory; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; @@ -365,6 +366,7 @@ public Collection createComponents(Client client, ClusterService cluster Auditor auditor = new Auditor(client, clusterService.getNodeName()); JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings); + JobConfigProvider jobConfigProvider = new JobConfigProvider(client, settings); UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(settings, client, clusterService, threadPool); JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, threadPool, client, notifier); @@ -420,6 +422,7 @@ public Collection createComponents(Client client, ClusterService cluster return Arrays.asList( mlLifeCycleService, jobResultsProvider, + jobConfigProvider, jobManager, autodetectProcessManager, new MlInitializationService(settings, threadPool, clusterService, client), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index 37d714d1777da..5df11f02a3610 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -79,7 +80,7 @@ public void clusterChanged(ClusterChangedEvent event) { if (Objects.equals(currentAssignment, previousAssignment)) { continue; } - if (OpenJobAction.TASK_NAME.equals(currentTask.getTaskName())) { + if (MlTasks.JOB_TASK_NAME.equals(currentTask.getTaskName())) { String jobId = ((OpenJobAction.JobParams) currentTask.getParams()).getJobId(); if (currentAssignment.getExecutorNode() == null) { auditor.warning(jobId, "No node found to open job. Reasons [" + currentAssignment.getExplanation() + "]"); @@ -87,7 +88,7 @@ public void clusterChanged(ClusterChangedEvent event) { DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); auditor.info(jobId, "Opening job on node [" + node.toString() + "]"); } - } else if (StartDatafeedAction.TASK_NAME.equals(currentTask.getTaskName())) { + } else if (MlTasks.DATAFEED_TASK_NAME.equals(currentTask.getTaskName())) { String datafeedId = ((StartDatafeedAction.DatafeedParams) currentTask.getParams()).getDatafeedId(); DatafeedConfig datafeedConfig = MlMetadata.getMlMetadata(event.state()).getDatafeed(datafeedId); if (currentAssignment.getExecutorNode() == null) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index d688db7fd0d1d..ebabcab59d674 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -19,18 +19,17 @@ import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; @@ -53,7 +52,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.MlMetaIndex; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; @@ -67,6 +65,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -99,22 +98,24 @@ public class TransportOpenJobAction extends TransportMasterNodeActioncheck job's version is supported * */ - static void validate(String jobId, MlMetadata mlMetadata) { - Job job = (mlMetadata == null) ? null : mlMetadata.getJobs().get(jobId); + static void validate(String jobId, Job job) { if (job == null) { throw ExceptionsHelper.missingJobException(jobId); } @@ -139,11 +139,14 @@ static void validate(String jobId, MlMetadata mlMetadata) { } } - static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String jobId, ClusterState clusterState, + static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String jobId, @Nullable Job job, + ClusterState clusterState, int maxConcurrentJobAllocations, int fallbackMaxNumberOfOpenJobs, - int maxMachineMemoryPercent, Logger logger) { - List unavailableIndices = verifyIndicesPrimaryShardsAreActive(jobId, clusterState); + int maxMachineMemoryPercent, + Logger logger) { + String resultsIndexName = job != null ? job.getResultsIndexName() : null; + List unavailableIndices = verifyIndicesPrimaryShardsAreActive(resultsIndexName, clusterState); if (unavailableIndices.size() != 0) { String reason = "Not opening job [" + jobId + "], because not all primary shards are active for the following indices [" + String.join(",", unavailableIndices) + "]"; @@ -153,12 +156,8 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j List reasons = new LinkedList<>(); long maxAvailableCount = Long.MIN_VALUE; - long maxAvailableMemory = Long.MIN_VALUE; DiscoveryNode minLoadedNodeByCount = null; - DiscoveryNode minLoadedNodeByMemory = null; - // Try to allocate jobs according to memory usage, but if that's not possible (maybe due to a mixed version cluster or maybe - // because of some weird OS problem) then fall back to the old mechanism of only considering numbers of assigned jobs - boolean allocateByMemory = true; + PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { Map nodeAttributes = node.getAttributes(); @@ -171,12 +170,9 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j continue; } - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - Job job = mlMetadata.getJobs().get(jobId); - Set compatibleJobTypes = Job.getCompatibleJobTypes(node.getVersion()); - if (compatibleJobTypes.contains(job.getJobType()) == false) { - String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + - "], because this node does not support jobs of type [" + job.getJobType() + "]"; + if (nodeSupportsMlJobs(node.getVersion()) == false) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + + "], because this node does not support machine learning jobs"; logger.trace(reason); reasons.add(reason); continue; @@ -191,22 +187,33 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j continue; } - if (jobHasRules(job) && node.getVersion().before(DetectionRule.VERSION_INTRODUCED)) { - String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because jobs using " + - "custom_rules require a node of version [" + DetectionRule.VERSION_INTRODUCED + "] or higher"; - logger.trace(reason); - reasons.add(reason); - continue; + if (job != null) { + Set compatibleJobTypes = Job.getCompatibleJobTypes(node.getVersion()); + if (compatibleJobTypes.contains(job.getJobType()) == false) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + + "], because this node does not support jobs of type [" + job.getJobType() + "]"; + logger.trace(reason); + reasons.add(reason); + continue; + } + + if (jobHasRules(job) && node.getVersion().before(DetectionRule.VERSION_INTRODUCED)) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because jobs using " + + "custom_rules require a node of version [" + DetectionRule.VERSION_INTRODUCED + "] or higher"; + logger.trace(reason); + reasons.add(reason); + continue; + } } + long numberOfAssignedJobs = 0; int numberOfAllocatingJobs = 0; - long assignedJobMemory = 0; + if (persistentTasks != null) { // find all the job tasks assigned to this node - Collection> assignedTasks = - persistentTasks.findTasks(OpenJobAction.TASK_NAME, - task -> node.getId().equals(task.getExecutorNode())); + Collection> assignedTasks = persistentTasks.findTasks( + MlTasks.JOB_TASK_NAME, task -> node.getId().equals(task.getExecutorNode())); for (PersistentTasksCustomMetaData.PersistentTask assignedTask : assignedTasks) { JobTaskState jobTaskState = (JobTaskState) assignedTask.getState(); JobState jobState; @@ -217,6 +224,7 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j } else { jobState = jobTaskState.getState(); if (jobTaskState.isStatusStale(assignedTask)) { + // the job is re-locating if (jobState == JobState.CLOSING) { // previous executor node failed while the job was closing - it won't // be reopened, so consider it CLOSED for resource usage purposes @@ -229,13 +237,9 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j } } } - // Don't count CLOSED or FAILED jobs, as they don't consume native memory if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) { + // Don't count CLOSED or FAILED jobs, as they don't consume native memory ++numberOfAssignedJobs; - String assignedJobId = ((OpenJobAction.JobParams) assignedTask.getParams()).getJobId(); - Job assignedJob = mlMetadata.getJobs().get(assignedJobId); - assert assignedJob != null; - assignedJobMemory += assignedJob.estimateMemoryFootprint(); } } } @@ -276,54 +280,10 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j maxAvailableCount = availableCount; minLoadedNodeByCount = node; } - - String machineMemoryStr = nodeAttributes.get(MachineLearning.MACHINE_MEMORY_NODE_ATTR); - long machineMemory = -1; - // TODO: remove leniency and reject the node if the attribute is null in 7.0 - if (machineMemoryStr != null) { - try { - machineMemory = Long.parseLong(machineMemoryStr); - } catch (NumberFormatException e) { - String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " + - MachineLearning.MACHINE_MEMORY_NODE_ATTR + " attribute [" + machineMemoryStr + "] is not a long"; - logger.trace(reason); - reasons.add(reason); - continue; - } - } - - if (allocateByMemory) { - if (machineMemory > 0) { - long maxMlMemory = machineMemory * maxMachineMemoryPercent / 100; - long estimatedMemoryFootprint = job.estimateMemoryFootprint(); - long availableMemory = maxMlMemory - assignedJobMemory; - if (estimatedMemoryFootprint > availableMemory) { - String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + - "], because this node has insufficient available memory. Available memory for ML [" + maxMlMemory + - "], memory required by existing jobs [" + assignedJobMemory + - "], estimated memory required for this job [" + estimatedMemoryFootprint + "]"; - logger.trace(reason); - reasons.add(reason); - continue; - } - - if (maxAvailableMemory < availableMemory) { - maxAvailableMemory = availableMemory; - minLoadedNodeByMemory = node; - } - } else { - // If we cannot get the available memory on any machine in - // the cluster, fall back to simply allocating by job count - allocateByMemory = false; - logger.debug("Falling back to allocating job [{}] by job counts because machine memory was not available for node [{}]", - jobId, nodeNameAndMlAttributes(node)); - } - } } - DiscoveryNode minLoadedNode = allocateByMemory ? minLoadedNodeByMemory : minLoadedNodeByCount; - if (minLoadedNode != null) { - logger.debug("selected node [{}] for job [{}]", minLoadedNode, jobId); - return new PersistentTasksCustomMetaData.Assignment(minLoadedNode.getId(), ""); + if (minLoadedNodeByCount != null) { + logger.debug("selected node [{}] for job [{}]", minLoadedNodeByCount, jobId); + return new PersistentTasksCustomMetaData.Assignment(minLoadedNodeByCount.getId(), ""); } else { String explanation = String.join("|", reasons); logger.debug("no node selected for job [{}], reasons [{}]", jobId, explanation); @@ -358,13 +318,15 @@ static String nodeNameAndMlAttributes(DiscoveryNode node) { return builder.toString(); } - static String[] indicesOfInterest(ClusterState clusterState, String job) { - String jobResultIndex = AnomalyDetectorsIndex.getPhysicalIndexFromState(clusterState, job); - return new String[]{AnomalyDetectorsIndex.jobStateIndexName(), jobResultIndex, MlMetaIndex.INDEX_NAME}; + static String[] indicesOfInterest(String resultsIndex) { + if (resultsIndex == null) { + return new String[]{AnomalyDetectorsIndex.jobStateIndexName(), MlMetaIndex.INDEX_NAME}; + } + return new String[]{AnomalyDetectorsIndex.jobStateIndexName(), resultsIndex, MlMetaIndex.INDEX_NAME}; } - static List verifyIndicesPrimaryShardsAreActive(String jobId, ClusterState clusterState) { - String[] indices = indicesOfInterest(clusterState, jobId); + static List verifyIndicesPrimaryShardsAreActive(String resultsIndex, ClusterState clusterState) { + String[] indices = indicesOfInterest(resultsIndex); List unavailableIndices = new ArrayList<>(indices.length); for (String index : indices) { // Indices are created on demand from templates. @@ -389,6 +351,10 @@ private static boolean nodeSupportsModelSnapshotVersion(DiscoveryNode node, Job return node.getVersion().onOrAfter(job.getModelSnapshotMinVersion()); } + private static boolean nodeSupportsMlJobs(Version nodeVersion) { + return nodeVersion.onOrAfter(Version.V_5_5_0); + } + private static boolean jobHasRules(Job job) { return job.getAnalysisConfig().getDetectors().stream().anyMatch(d -> d.getRules().isEmpty() == false); } @@ -466,7 +432,7 @@ protected void masterOperation(OpenJobAction.Request request, ClusterState state OpenJobAction.JobParams jobParams = request.getJobParams(); if (licenseState.isMachineLearningAllowed()) { - // Step 6. Clear job finished time once the job is started and respond + // Clear job finished time once the job is started and respond ActionListener clearJobFinishTime = ActionListener.wrap( response -> { if (response.isAcknowledged()) { @@ -478,7 +444,7 @@ protected void masterOperation(OpenJobAction.Request request, ClusterState state listener::onFailure ); - // Step 5. Wait for job to be started + // Wait for job to be started ActionListener> waitForJobToStart = new ActionListener>() { @Override @@ -496,17 +462,20 @@ public void onFailure(Exception e) { } }; - // Step 4. Start job task + // Start job task ActionListener establishedMemoryUpdateListener = ActionListener.wrap( - response -> persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()), - OpenJobAction.TASK_NAME, jobParams, waitForJobToStart), + response -> { + persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()), + MlTasks.JOB_TASK_NAME, jobParams, waitForJobToStart); + }, listener::onFailure ); - // Step 3. Update established model memory for pre-6.1 jobs that haven't had it set + // Update established model memory for pre-6.1 jobs that haven't had it set + // and increase the model memory limit for 6.1 - 6.3 jobs ActionListener missingMappingsListener = ActionListener.wrap( response -> { - Job job = MlMetadata.getMlMetadata(clusterService.state()).getJobs().get(jobParams.getJobId()); + Job job = jobParams.getJob(); if (job != null) { Version jobVersion = job.getJobVersion(); Long jobEstablishedModelMemory = job.getEstablishedModelMemory(); @@ -533,7 +502,7 @@ public void onFailure(Exception e) { }, listener::onFailure ); - // Step 2. Try adding state doc mapping + // Try adding state doc mapping ActionListener resultsPutMappingHandler = ActionListener.wrap( response -> { addDocMappingIfMissing(AnomalyDetectorsIndex.jobStateIndexName(), ElasticsearchMappings::stateMapping, @@ -541,9 +510,21 @@ public void onFailure(Exception e) { }, listener::onFailure ); - // Step 1. Try adding results doc mapping - addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobParams.getJobId()), ElasticsearchMappings::resultsMapping, - state, resultsPutMappingHandler); + // Get the job config + jobConfigProvider.getJob(jobParams.getJobId(), ActionListener.wrap( + builder -> { + try { + jobParams.setJob(builder.build()); + + // Try adding results doc mapping + addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobParams.getJobId()), + ElasticsearchMappings::resultsMapping, state, resultsPutMappingHandler); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + )); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); } @@ -582,34 +563,18 @@ public void onTimeout(TimeValue timeout) { } private void clearJobFinishedTime(String jobId, ActionListener listener) { - clusterService.submitStateUpdateTask("clearing-job-finish-time-for-" + jobId, new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(currentState); - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(mlMetadata); - Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId)); - jobBuilder.setFinishedTime(null); - - mlMetadataBuilder.putJob(jobBuilder.build(), true); - ClusterState.Builder builder = ClusterState.builder(currentState); - return builder.metaData(new MetaData.Builder(currentState.metaData()) - .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build())) - .build(); - } - - @Override - public void onFailure(String source, Exception e) { - logger.error("[" + jobId + "] Failed to clear finished_time; source [" + source + "]", e); - listener.onResponse(new AcknowledgedResponse(true)); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, - ClusterState newState) { - listener.onResponse(new AcknowledgedResponse(true)); - } - }); + JobUpdate update = new JobUpdate.Builder(jobId).setClearJobFinishTime(true).build(); + + jobConfigProvider.updateJob(jobId, update, null, ActionListener.wrap( + job -> listener.onResponse(new AcknowledgedResponse(true)), + e -> { + logger.error("[" + jobId + "] Failed to clear finished_time", e); + // Not a critical error so continue + listener.onResponse(new AcknowledgedResponse(true)); + } + )); } + private void cancelJobStart(PersistentTasksCustomMetaData.PersistentTask persistentTask, Exception exception, ActionListener listener) { persistentTasksService.sendRemoveRequest(persistentTask.getId(), @@ -690,7 +655,7 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterService, AutodetectProcessManager autodetectProcessManager) { - super(settings, OpenJobAction.TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); + super(settings, MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); this.autodetectProcessManager = autodetectProcessManager; this.fallbackMaxNumberOfOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(settings); this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); @@ -705,12 +670,8 @@ public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterS @Override public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobParams params, ClusterState clusterState) { - PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(params.getJobId(), - clusterState, - maxConcurrentJobAllocations, - fallbackMaxNumberOfOpenJobs, - maxMachineMemoryPercent, - logger); + PersistentTasksCustomMetaData.Assignment assignment =selectLeastLoadedMlNode(params.getJobId(), params.getJob(), clusterState, + maxConcurrentJobAllocations, fallbackMaxNumberOfOpenJobs, maxMachineMemoryPercent, logger); if (assignment.getExecutorNode() == null) { int numMlNodes = 0; for(DiscoveryNode node : clusterState.getNodes()) { @@ -729,11 +690,10 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobP @Override public void validate(OpenJobAction.JobParams params, ClusterState clusterState) { - TransportOpenJobAction.validate(params.getJobId(), MlMetadata.getMlMetadata(clusterState)); + TransportOpenJobAction.validate(params.getJobId(), params.getJob()); // If we already know that we can't find an ml node because all ml nodes are running at capacity or // simply because there are no ml nodes in the cluster then we fail quickly here: - PersistentTasksCustomMetaData.Assignment assignment = getAssignment(params, clusterState); if (assignment.getExecutorNode() == null && assignment.equals(AWAITING_LAZY_ASSIGNMENT) == false) { throw makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanation()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index f3f4a771443ef..578f4ee5f983b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -173,7 +173,7 @@ private void createDataExtractor(Job job, DatafeedConfig datafeed, StartDatafeed DataExtractorFactory.create(client, datafeed, job, ActionListener.wrap( dataExtractorFactory -> persistentTasksService.sendStartRequest(MlTasks.datafeedTaskId(params.getDatafeedId()), - StartDatafeedAction.TASK_NAME, params, listener) + MlTasks.DATAFEED_TASK_NAME, params, listener) , listener::onFailure)); } @@ -272,7 +272,7 @@ public static class StartDatafeedPersistentTasksExecutor extends PersistentTasks private final IndexNameExpressionResolver resolver; public StartDatafeedPersistentTasksExecutor(Settings settings, DatafeedManager datafeedManager) { - super(settings, StartDatafeedAction.TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); + super(settings, MlTasks.DATAFEED_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); this.datafeedManager = datafeedManager; this.resolver = new IndexNameExpressionResolver(settings); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index ef593417a19e3..b30338cd52c38 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -16,6 +16,9 @@ import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -145,6 +148,51 @@ public void onFailure(Exception e) { }, client::get); } + /** + * Get the list anomaly detector jobs specified by {@code jobIds}. + * + * WARNING: errors are silently ignored, if a job is not found a + * {@code ResourceNotFoundException} is not thrown. Only found + * jobs are returned, this size of the returned jobs list could + * be different to the size of the requested ids list. + * + * @param jobIds The jobs to get + * @param listener Jobs listener + */ + public void getJobs(List jobIds, ActionListener> listener) { + MultiGetRequest multiGetRequest = new MultiGetRequest(); + jobIds.forEach(jobId -> multiGetRequest.add(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId))); + + List jobs = new ArrayList<>(); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, multiGetRequest, new ActionListener() { + @Override + public void onResponse(MultiGetResponse multiGetResponse) { + + MultiGetItemResponse[] responses = multiGetResponse.getResponses(); + for (MultiGetItemResponse response : responses) { + GetResponse getResponse = response.getResponse(); + if (getResponse.isExists()) { + BytesReference source = getResponse.getSourceAsBytesRef(); + try { + Job.Builder job = parseJobLenientlyFromSource(source); + jobs.add(job); + } catch (IOException e) { + logger.error("Error parsing job configuration [" + response.getId() + "]"); + } + } + } + + listener.onResponse(jobs); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, client::multiGet); + } + /** * Delete the anomaly detector job config document * diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java index 82478fbf5d337..c7ca2ff805eba 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java @@ -303,7 +303,7 @@ public void testUpdateDatafeed_failBecauseDatafeedIsNotStopped() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); StartDatafeedAction.DatafeedParams params = new StartDatafeedAction.DatafeedParams(datafeedConfig1.getId(), 0L); - tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT); + tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), MlTasks.DATAFEED_TASK_NAME, params, INITIAL_ASSIGNMENT); PersistentTasksCustomMetaData tasksInProgress = tasksBuilder.build(); DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId()); @@ -385,7 +385,7 @@ public void testRemoveDatafeed_failBecauseDatafeedStarted() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); StartDatafeedAction.DatafeedParams params = new StartDatafeedAction.DatafeedParams("datafeed1", 0L); - tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT); + tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), MlTasks.DATAFEED_TASK_NAME, params, INITIAL_ASSIGNMENT); PersistentTasksCustomMetaData tasksInProgress = tasksBuilder.build(); MlMetadata.Builder builder2 = new MlMetadata.Builder(result); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 98b84ed81137b..879e4c3fa18af 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -311,7 +311,7 @@ public void testBuildWaitForCloseRequest() { public static void addTask(String datafeedId, long startTime, String nodeId, DatafeedState state, PersistentTasksCustomMetaData.Builder tasks) { - tasks.addTask(MlTasks.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME, + tasks.addTask(MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams(datafeedId, startTime), new Assignment(nodeId, "test assignment")); tasks.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 4dd41363b73fe..e18edbbbd1b25 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xpack.core.ml.MlMetaIndex; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -63,7 +62,6 @@ import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; -import java.util.function.Function; import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.hamcrest.Matchers.containsString; @@ -74,36 +72,28 @@ public class TransportOpenJobActionTests extends ESTestCase { public void testValidate_jobMissing() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(buildJobBuilder("job_id1").build(), false); - expectThrows(ResourceNotFoundException.class, () -> TransportOpenJobAction.validate("job_id2", mlBuilder.build())); + expectThrows(ResourceNotFoundException.class, () -> TransportOpenJobAction.validate("job_id2", null)); } public void testValidate_jobMarkedAsDeleting() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); Job.Builder jobBuilder = buildJobBuilder("job_id"); jobBuilder.setDeleting(true); - mlBuilder.putJob(jobBuilder.build(), false); Exception e = expectThrows(ElasticsearchStatusException.class, - () -> TransportOpenJobAction.validate("job_id", mlBuilder.build())); - assertEquals("Cannot open job [job_id] because it is being deleted", e.getMessage()); + () -> TransportOpenJobAction.validate("job_id", jobBuilder.build())); + assertEquals("Cannot open job [job_id] because it has been marked as deleted", e.getMessage()); } public void testValidate_jobWithoutVersion() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); Job.Builder jobBuilder = buildJobBuilder("job_id"); - mlBuilder.putJob(jobBuilder.build(), false); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> TransportOpenJobAction.validate("job_id", mlBuilder.build())); + () -> TransportOpenJobAction.validate("job_id", jobBuilder.build())); assertEquals("Cannot open job [job_id] because jobs created prior to version 5.5 are not supported", e.getMessage()); assertEquals(RestStatus.BAD_REQUEST, e.status()); } public void testValidate_givenValidJob() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); Job.Builder jobBuilder = buildJobBuilder("job_id"); - mlBuilder.putJob(jobBuilder.build(new Date()), false); - TransportOpenJobAction.validate("job_id", mlBuilder.build()); + TransportOpenJobAction.validate("job_id", jobBuilder.build(new Date())); } public void testSelectLeastLoadedMlNode_byCount() { @@ -126,94 +116,21 @@ public void testSelectLeastLoadedMlNode_byCount() { PersistentTasksCustomMetaData tasks = tasksBuilder.build(); ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); - MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4"); cs.nodes(nodes); - metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); - cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, 10, 30, logger); - assertEquals("", result.getExplanation()); - assertEquals("_node_id3", result.getExecutorNode()); - } - - public void testSelectLeastLoadedMlNode_byMemory() { - Map nodeAttr = new HashMap<>(); - nodeAttr.put(MachineLearning.ML_ENABLED_NODE_ATTR, "true"); - nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "16000000000"); - DiscoveryNodes nodes = DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .build(); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id1", "_node_id1", JobState.fromString("opened"), tasksBuilder); - addJobTask("job_id2", "_node_id2", JobState.fromString("opened"), tasksBuilder); - addJobTask("job_id3", "_node_id2", JobState.fromString("opened"), tasksBuilder); - addJobTask("job_id4", "_node_id3", JobState.fromString("opened"), tasksBuilder); - PersistentTasksCustomMetaData tasks = tasksBuilder.build(); - - ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobId -> { - // remember we add 100MB for the process overhead, so these model memory - // limits correspond to estimated footprints of 102MB and 205MB - long jobSize = (jobId.equals("job_id2") || jobId.equals("job_id3")) ? 2 : 105; - return BaseMlIntegTestCase.createFareQuoteJob(jobId, new ByteSizeValue(jobSize, ByteSizeUnit.MB)).build(new Date()); - }, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5"); - cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id5", cs.build(), 2, 10, 30, logger); - assertEquals("", result.getExplanation()); - assertEquals("_node_id2", result.getExecutorNode()); - } - public void testSelectLeastLoadedMlNode_byMemoryWithFailedJobs() { - Map nodeAttr = new HashMap<>(); - nodeAttr.put(MachineLearning.ML_ENABLED_NODE_ATTR, "true"); - // this leaves just under 300MB per node available for ML jobs - nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000"); - DiscoveryNodes nodes = DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302), - nodeAttr, Collections.emptySet(), Version.CURRENT)) - .build(); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id1", "_node_id1", JobState.fromString("failed"), tasksBuilder); - addJobTask("job_id2", "_node_id2", JobState.fromString("failed"), tasksBuilder); - addJobTask("job_id3", "_node_id3", JobState.fromString("failed"), tasksBuilder); - PersistentTasksCustomMetaData tasks = tasksBuilder.build(); + Job.Builder jobBuilder = buildJobBuilder("job_id4"); + jobBuilder.setJobVersion(Version.CURRENT); - ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); - MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobId -> { - // remember we add 100MB for the process overhead, so this model - // memory limit corresponds to a job size of 250MB - return BaseMlIntegTestCase.createFareQuoteJob(jobId, new ByteSizeValue(150, ByteSizeUnit.MB)).build(new Date()); - }, "job_id1", "job_id2", "job_id3", "job_id4"); - cs.nodes(nodes); - metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); - cs.metaData(metaData); - cs.routingTable(routingTable.build()); - // if the memory of the failed jobs is wrongly included in the calculation then this job will not be allocated - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", jobBuilder.build(), + cs.build(), 2, 10, 30, logger); assertEquals("", result.getExplanation()); - assertNotNull(result.getExecutorNode()); + assertEquals("_node_id3", result.getExecutorNode()); } + public void testSelectLeastLoadedMlNode_maxCapacity() { int numNodes = randomIntBetween(1, 10); int maxRunningJobsPerNode = randomIntBetween(1, 100); @@ -237,13 +154,14 @@ public void testSelectLeastLoadedMlNode_maxCapacity() { ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobIds); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id0", cs.build(), 2, maxRunningJobsPerNode, 30, logger); + + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id0", new ByteSizeValue(150, ByteSizeUnit.MB)).build(new Date()); + + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id0", job, cs.build(), 2, + maxRunningJobsPerNode, 30, logger); assertNull(result.getExecutorNode()); assertTrue(result.getExplanation().contains("because this node is full. Number of opened jobs [" + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]")); @@ -263,13 +181,13 @@ public void testSelectLeastLoadedMlNode_noMlNodes() { ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, "job_id1", "job_id2"); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, 10, 30, logger); + + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id2", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); + + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 2, 10, 30, logger); assertTrue(result.getExplanation().contains("because this node isn't a ml node")); assertNull(result.getExecutorNode()); } @@ -297,14 +215,13 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.nodes(nodes); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7"); - csBuilder.routingTable(routingTable.build()); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); csBuilder.metaData(metaData); + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id6", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); + ClusterState cs = csBuilder.build(); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", cs, 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 2, 10, 30, logger); assertEquals("_node_id3", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -314,7 +231,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -325,7 +242,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); assertNull("no node selected, because stale task", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -336,7 +253,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); assertNull("no node selected, because null state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -367,15 +284,14 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.nodes(nodes); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, "job_id1", "job_id2", "job_id3", "job_id4", "job_id5", "job_id6", "job_id7", "job_id8"); - csBuilder.routingTable(routingTable.build()); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); csBuilder.metaData(metaData); ClusterState cs = csBuilder.build(); + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); + // Allocation won't be possible if the stale failed job is treated as opening - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", cs, 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); assertEquals("_node_id1", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -385,7 +301,7 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 2, 10, 30, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -406,21 +322,17 @@ public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() { ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - Function incompatibleJobCreator = jobId -> { - Job job = mock(Job.class); - when(job.getId()).thenReturn(jobId); - when(job.getJobVersion()).thenReturn(Version.CURRENT); - when(job.getJobType()).thenReturn("incompatible_type"); - when(job.getResultsIndexName()).thenReturn("shared"); - return job; - }; - addJobAndIndices(metaData, routingTable, incompatibleJobCreator, "incompatible_type_job"); + + Job job = mock(Job.class); + when(job.getId()).thenReturn("incompatible_type_job"); + when(job.getJobVersion()).thenReturn(Version.CURRENT); + when(job.getJobType()).thenReturn("incompatible_type"); + when(job.getResultsIndexName()).thenReturn("shared"); + cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, logger); assertThat(result.getExplanation(), containsString("because this node does not support jobs of type [incompatible_type]")); assertNull(result.getExecutorNode()); } @@ -441,19 +353,14 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobId -> BaseMlIntegTestCase.createFareQuoteJob(jobId) - .setModelSnapshotId("incompatible_snapshot") - .setModelSnapshotMinVersion(Version.V_6_3_0) - .build(new Date()), "job_with_incompatible_model_snapshot"); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_incompatible_model_snapshot", cs.build(), - 2, 10, 30, logger); - assertThat(result.getExplanation(), containsString( - "because the job's model snapshot requires a node of version [6.3.0] or higher")); + + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); + + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, logger); + assertThat(result.getExplanation(), containsString("because this node does not support machine learning jobs")); assertNull(result.getExecutorNode()); } @@ -473,14 +380,12 @@ public void testSelectLeastLoadedMlNode_jobWithRulesButNoNodeMeetsRequiredVersio ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobWithRulesCreator(), "job_with_rules"); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", cs.build(), - 2, 10, 30, logger); + + Job job = jobWithRules("job_with_rules"); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, logger); assertThat(result.getExplanation(), containsString( "because jobs using custom_rules require a node of version [6.4.0] or higher")); assertNull(result.getExecutorNode()); @@ -502,33 +407,31 @@ public void testSelectLeastLoadedMlNode_jobWithRulesAndNodeMeetsRequiredVersion( ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, jobWithRulesCreator(), "job_with_rules"); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - cs.routingTable(routingTable.build()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", cs.build(), - 2, 10, 30, logger); + + Job job = jobWithRules("job_with_rules"); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, logger); assertNotNull(result.getExecutorNode()); } public void testVerifyIndicesPrimaryShardsAreActive() { MetaData.Builder metaData = MetaData.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); - addJobAndIndices(metaData, routingTable, "job_id"); + addIndices(metaData, routingTable); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.routingTable(routingTable.build()); csBuilder.metaData(metaData); ClusterState cs = csBuilder.build(); - assertEquals(0, TransportOpenJobAction.verifyIndicesPrimaryShardsAreActive("job_id", cs).size()); + assertEquals(0, TransportOpenJobAction.verifyIndicesPrimaryShardsAreActive(".ml-anomalies-shared", cs).size()); metaData = new MetaData.Builder(cs.metaData()); routingTable = new RoutingTable.Builder(cs.routingTable()); - String indexToRemove = randomFrom(TransportOpenJobAction.indicesOfInterest(cs, "job_id")); + String indexToRemove = randomFrom(TransportOpenJobAction.indicesOfInterest(".ml-anomalies-shared")); if (randomBoolean()) { routingTable.remove(indexToRemove); } else { @@ -543,7 +446,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { csBuilder.routingTable(routingTable.build()); csBuilder.metaData(metaData); - List result = TransportOpenJobAction.verifyIndicesPrimaryShardsAreActive("job_id", csBuilder.build()); + List result = TransportOpenJobAction.verifyIndicesPrimaryShardsAreActive(".ml-anomalies-shared", csBuilder.build()); assertEquals(1, result.size()); assertEquals(indexToRemove, result.get(0)); } @@ -647,20 +550,14 @@ public void testJobTaskMatcherMatch() { } public static void addJobTask(String jobId, String nodeId, JobState jobState, PersistentTasksCustomMetaData.Builder builder) { - builder.addTask(MlTasks.jobTaskId(jobId), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams(jobId), + builder.addTask(MlTasks.jobTaskId(jobId), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId), new Assignment(nodeId, "test assignment")); if (jobState != null) { builder.updateTaskState(MlTasks.jobTaskId(jobId), new JobTaskState(jobState, builder.getLastAllocationId())); } } - private void addJobAndIndices(MetaData.Builder metaData, RoutingTable.Builder routingTable, String... jobIds) { - addJobAndIndices(metaData, routingTable, jobId -> - BaseMlIntegTestCase.createFareQuoteJob(jobId, new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()), jobIds); - } - - private void addJobAndIndices(MetaData.Builder metaData, RoutingTable.Builder routingTable, Function jobCreator, - String... jobIds) { + private void addIndices(MetaData.Builder metaData, RoutingTable.Builder routingTable) { List indices = new ArrayList<>(); indices.add(AnomalyDetectorsIndex.jobStateIndexName()); indices.add(MlMetaIndex.INDEX_NAME); @@ -683,13 +580,6 @@ private void addJobAndIndices(MetaData.Builder metaData, RoutingTable.Builder ro routingTable.add(IndexRoutingTable.builder(index) .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build())); } - - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - for (String jobId : jobIds) { - Job job = jobCreator.apply(jobId); - mlMetadata.putJob(job, false); - } - metaData.putCustom(MlMetadata.TYPE, mlMetadata.build()); } private ClusterState getClusterStateWithMappingsWithMetaData(Map namesAndVersions) throws IOException { @@ -728,21 +618,19 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map return csBuilder.build(); } - private static Function jobWithRulesCreator() { - return jobId -> { - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList( - new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.LT, 100.0) - )).build(); - - Detector.Builder detector = new Detector.Builder("count", null); - detector.setRules(Collections.singletonList(rule)); - AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); - DataDescription.Builder dataDescription = new DataDescription.Builder(); - Job.Builder job = new Job.Builder(jobId); - job.setAnalysisConfig(analysisConfig); - job.setDataDescription(dataDescription); - return job.build(new Date()); - }; + private static Job jobWithRules(String jobId) { + DetectionRule rule = new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.LT, 100.0) + )).build(); + + Detector.Builder detector = new Detector.Builder("count", null); + detector.setRules(Collections.singletonList(rule)); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + Job.Builder job = new Job.Builder(jobId); + job.setAnalysisConfig(analysisConfig); + job.setDataDescription(dataDescription); + return job.build(new Date()); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java index a15c0e97b97f1..d8b1d28153688 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java @@ -29,7 +29,7 @@ public class TransportStopDatafeedActionTests extends ESTestCase { public void testValidate() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME, + tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo", 0L), new PersistentTasksCustomMetaData.Assignment("node_id", "")); tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED); tasksBuilder.build(); @@ -118,7 +118,7 @@ public void testResolveDataFeedIds_GivenAll() { public static void addTask(String datafeedId, long startTime, String nodeId, DatafeedState state, PersistentTasksCustomMetaData.Builder taskBuilder) { - taskBuilder.addTask(MlTasks.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME, + taskBuilder.addTask(MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams(datafeedId, startTime), new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment")); taskBuilder.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index c86db02ca807e..1303e2c747773 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -263,7 +263,7 @@ public void testMaxConcurrentJobAllocations() throws Exception { } for (DiscoveryNode node : event.state().nodes()) { - Collection> foundTasks = tasks.findTasks(OpenJobAction.TASK_NAME, task -> { + Collection> foundTasks = tasks.findTasks(MlTasks.JOB_TASK_NAME, task -> { JobTaskState jobTaskState = (JobTaskState) task.getState(); return node.getId().equals(task.getExecutorNode()) && (jobTaskState == null || jobTaskState.isStatusStale(task)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index 3456290745a05..ba0f5520e0740 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -46,6 +46,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsInstanceOf.instanceOf; @@ -130,7 +131,6 @@ public void testCrud() throws InterruptedException { AtomicReference getJobResponseHolder = new AtomicReference<>(); blockingCall(actionListener -> jobConfigProvider.getJob(jobId, actionListener), getJobResponseHolder, exceptionHolder); assertNull(exceptionHolder.get()); - assertEquals(newJob, getJobResponseHolder.get().build()); // Update Job @@ -170,6 +170,23 @@ public void testCrud() throws InterruptedException { assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); } + public void testGetJobs() throws Exception { + putJob(createJob("nginx", null)); + putJob(createJob("tomcat", null)); + putJob(createJob("mysql", null)); + + List jobsToGet = Arrays.asList("nginx", "tomcat", "unknown-job"); + + AtomicReference> jobsHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.getJobs(jobsToGet, actionListener), jobsHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertNotNull(jobsHolder.get()); + assertThat(jobsHolder.get(), hasSize(2)); + List foundIds = jobsHolder.get().stream().map(Job.Builder::getId).collect(Collectors.toList()); + assertThat(foundIds, containsInAnyOrder("nginx", "tomcat")); + } + public void testUpdateWithAValidationError() throws Exception { final String jobId = "bad-update-job"; From c75a7d48489268f1acbbf9ff8fd3ef4a732a741a Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 1 Oct 2018 12:34:12 +0100 Subject: [PATCH 07/57] [ML] Return missing job error when .ml-config is does not exist (#34177) --- .../persistence/DatafeedConfigProvider.java | 10 ++++- .../xpack/ml/job/JobManager.java | 13 +++++- .../ml/job/persistence/JobConfigProvider.java | 41 +++++++++++++++---- .../integration/DatafeedConfigProviderIT.java | 17 ++++++-- .../ml/integration/JobConfigProviderIT.java | 34 +++++++++------ 5 files changed, 87 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index f0402ed869224..22912e9afc795 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -136,6 +137,9 @@ public void putDatafeedConfig(DatafeedConfig config, Map headers * If the datafeed document is missing a {@code ResourceNotFoundException} * is returned via the listener. * + * If the .ml-config index does not exist it is treated as a missing datafeed + * error. + * * @param datafeedId The datafeed ID * @param datafeedConfigListener The config listener */ @@ -154,7 +158,11 @@ public void onResponse(GetResponse getResponse) { } @Override public void onFailure(Exception e) { - datafeedConfigListener.onFailure(e); + if (e.getClass() == IndexNotFoundException.class) { + datafeedConfigListener.onFailure(ExceptionsHelper.missingDatafeedException(datafeedId)); + } else { + datafeedConfigListener.onFailure(e); + } } }); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 170de42a78102..0393f65bcaba7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -120,7 +120,7 @@ private void setMaxModelMemoryLimit(ByteSizeValue maxModelMemoryLimit) { } public void jobExists(String jobId, ActionListener listener) { - jobConfigProvider.checkJobExists(jobId, listener); + jobConfigProvider.jobExists(jobId, true, listener); } /** @@ -281,7 +281,16 @@ public void onFailure(Exception e) { actionListener::onFailure ); - jobResultsProvider.checkForLeftOverDocuments(job, checkForLeftOverDocs); + jobConfigProvider.jobExists(job.getId(), false, ActionListener.wrap( + jobExists -> { + if (jobExists) { + actionListener.onFailure(ExceptionsHelper.jobAlreadyExists(job.getId())); + } else { + jobResultsProvider.checkForLeftOverDocuments(job, checkForLeftOverDocs); + } + }, + actionListener::onFailure + )); } public void updateJob(UpdateJobAction.Request request, ActionListener actionListener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index b30338cd52c38..feab1e84a0146 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -122,6 +123,9 @@ public void putJob(Job job, ActionListener listener) { * If the job is missing a {@code ResourceNotFoundException} is returned * via the listener. * + * If the .ml-config index does not exist it is treated as a missing job + * error. + * * @param jobId The job ID * @param jobListener Job listener */ @@ -143,7 +147,11 @@ public void onResponse(GetResponse getResponse) { @Override public void onFailure(Exception e) { - jobListener.onFailure(e); + if (e.getClass() == IndexNotFoundException.class) { + jobListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + jobListener.onFailure(e); + } } }, client::get); } @@ -368,14 +376,19 @@ private void indexUpdatedJob(Job updatedJob, long version, ActionListener u /** * Check a job exists. A job exists if it has a configuration document. + * If the .ml-config index does not exist it is treated as a missing job + * error. * - * If the job does not exist a ResourceNotFoundException is returned to the listener, - * FALSE will never be returned only TRUE or ResourceNotFoundException + * Depending on the value of {@code errorIfMissing} if the job does not + * exist a ResourceNotFoundException is returned to the listener, + * otherwise false is returned in the response. * - * @param jobId The jobId to check - * @param listener Exists listener + * @param jobId The jobId to check + * @param errorIfMissing If true and the job is missing the listener fails with + * a ResourceNotFoundException else false is returned. + * @param listener Exists listener */ - public void checkJobExists(String jobId, ActionListener listener) { + public void jobExists(String jobId, boolean errorIfMissing, ActionListener listener) { GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); getRequest.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); @@ -384,7 +397,11 @@ public void checkJobExists(String jobId, ActionListener listener) { @Override public void onResponse(GetResponse getResponse) { if (getResponse.isExists() == false) { - listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + if (errorIfMissing) { + listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + listener.onResponse(Boolean.FALSE); + } } else { listener.onResponse(Boolean.TRUE); } @@ -392,7 +409,15 @@ public void onResponse(GetResponse getResponse) { @Override public void onFailure(Exception e) { - listener.onFailure(e); + if (e.getClass() == IndexNotFoundException.class) { + if (errorIfMissing) { + listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + listener.onResponse(Boolean.FALSE); + } + } else { + listener.onFailure(e); + } } }); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index 3f70baf4771c3..c4dd8c19ea611 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -108,6 +108,15 @@ public void testCrud() throws InterruptedException { assertEquals(DocWriteResponse.Result.DELETED, deleteResponseHolder.get().getResult()); } + public void testGetDatafeedConfig_missing() throws InterruptedException { + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference configBuilderHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.getDatafeedConfig("missing", actionListener), + configBuilderHolder, exceptionHolder); + assertNull(configBuilderHolder.get()); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); + } + public void testMultipleCreateAndDeletes() throws InterruptedException { String datafeedId = "df2"; @@ -127,7 +136,7 @@ public void testMultipleCreateAndDeletes() throws InterruptedException { indexResponseHolder, exceptionHolder); assertNull(indexResponseHolder.get()); assertThat(exceptionHolder.get(), instanceOf(ResourceAlreadyExistsException.class)); - assertEquals("A datafeed with Id [df2] already exists", exceptionHolder.get().getMessage()); + assertEquals("A datafeed with id [df2] already exists", exceptionHolder.get().getMessage()); // delete exceptionHolder.set(null); @@ -142,7 +151,7 @@ public void testMultipleCreateAndDeletes() throws InterruptedException { blockingCall(actionListener -> datafeedConfigProvider.deleteDatafeedConfig(datafeedId, actionListener), deleteResponseHolder, exceptionHolder); assertNull(deleteResponseHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); } public void testUpdateWhenApplyingTheUpdateThrows() throws Exception { @@ -202,7 +211,7 @@ public void testAllowNoDatafeeds() throws InterruptedException { assertNull(datafeedIdsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), containsString("No datafeed with id [*] exists")); exceptionHolder.set(null); @@ -217,7 +226,7 @@ public void testAllowNoDatafeeds() throws InterruptedException { assertNull(datafeedsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), IsInstanceOf.instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), containsString("No datafeed with id [*] exists")); exceptionHolder.set(null); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index ba0f5520e0740..63f67c37dd944 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -68,18 +68,25 @@ public void testGetMissingJob() throws InterruptedException { assertNull(jobHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); } public void testCheckJobExists() throws InterruptedException { AtomicReference jobExistsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.checkJobExists("missing", actionListener), jobExistsHolder, exceptionHolder); - - assertNull(jobExistsHolder.get()); - assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + boolean throwIfMissing = randomBoolean(); + blockingCall(actionListener -> + jobConfigProvider.jobExists("missing", throwIfMissing, actionListener), jobExistsHolder, exceptionHolder); + + if (throwIfMissing) { + assertNull(jobExistsHolder.get()); + assertNotNull(exceptionHolder.get()); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); + } else { + assertFalse(jobExistsHolder.get()); + assertNull(exceptionHolder.get()); + } AtomicReference indexResponseHolder = new AtomicReference<>(); @@ -88,7 +95,8 @@ public void testCheckJobExists() throws InterruptedException { blockingCall(actionListener -> jobConfigProvider.putJob(job, actionListener), indexResponseHolder, exceptionHolder); exceptionHolder.set(null); - blockingCall(actionListener -> jobConfigProvider.checkJobExists("existing-job", actionListener), jobExistsHolder, exceptionHolder); + blockingCall(actionListener -> + jobConfigProvider.jobExists("existing-job", throwIfMissing, actionListener), jobExistsHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertNotNull(jobExistsHolder.get()); assertTrue(jobExistsHolder.get()); @@ -159,7 +167,7 @@ public void testCrud() throws InterruptedException { getJobResponseHolder.set(null); blockingCall(actionListener -> jobConfigProvider.getJob(jobId, actionListener), getJobResponseHolder, exceptionHolder); assertNull(getJobResponseHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); // Delete deleted job deleteJobResponseHolder.set(null); @@ -167,7 +175,7 @@ public void testCrud() throws InterruptedException { blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, actionListener), deleteJobResponseHolder, exceptionHolder); assertNull(deleteJobResponseHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); } public void testGetJobs() throws Exception { @@ -263,7 +271,7 @@ public void testAllowNoJobs() throws InterruptedException { assertNull(jobIdsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); exceptionHolder.set(null); @@ -278,7 +286,7 @@ public void testAllowNoJobs() throws InterruptedException { assertNull(jobsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); exceptionHolder.set(null); @@ -315,7 +323,7 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { jobIdsHolder, exceptionHolder); assertNull(jobIdsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), equalTo("No known job with id 'missing1,missing2'")); // Job builders @@ -344,7 +352,7 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { jobsHolder, exceptionHolder); assertNull(jobsHolder.get()); assertNotNull(exceptionHolder.get()); - assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class)); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); assertThat(exceptionHolder.get().getMessage(), equalTo("No known job with id 'missing1,missing2'")); } From e3821be70022d8632054009ff3d68e8997d0c35d Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 18 Oct 2018 10:36:36 +0100 Subject: [PATCH 08/57] [ML] Close job in index (#34217) --- .../xpack/core/ml/job/config/JobUpdate.java | 2 +- .../xpack/ml/MachineLearning.java | 3 + .../ml/action/TransportCloseJobAction.java | 290 ++++++++------- .../ml/action/TransportPutDatafeedAction.java | 3 +- .../action/TransportUpdateDatafeedAction.java | 3 +- .../persistence/DatafeedConfigProvider.java | 19 +- .../action/TransportCloseJobActionTests.java | 351 ++++++++---------- .../integration/DatafeedConfigProviderIT.java | 10 +- 8 files changed, 341 insertions(+), 340 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 5e947d4fe8c28..37fbf3dab14d8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -489,7 +489,7 @@ && updatesDetectors(job) == false && (modelSnapshotMinVersion == null || Objects.equals(modelSnapshotMinVersion, job.getModelSnapshotMinVersion())) && (establishedModelMemory == null || Objects.equals(establishedModelMemory, job.getEstablishedModelMemory())) && (jobVersion == null || Objects.equals(jobVersion, job.getJobVersion())) - && (clearJobFinishTime == false || job.getFinishedTime() == null); + && ((clearJobFinishTime == null || clearJobFinishTime == false) || job.getFinishedTime() == null); } boolean updatesDetectors(Job job) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index d316ad67b0692..a13b705f12408 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -161,6 +161,7 @@ import org.elasticsearch.xpack.ml.action.TransportValidateJobConfigAction; import org.elasticsearch.xpack.ml.datafeed.DatafeedJobBuilder; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.UpdateJobProcessNotifier; import org.elasticsearch.xpack.ml.job.categorization.MlClassicTokenizer; @@ -367,6 +368,7 @@ public Collection createComponents(Client client, ClusterService cluster Auditor auditor = new Auditor(client, clusterService.getNodeName()); JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings); JobConfigProvider jobConfigProvider = new JobConfigProvider(client, settings); + DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry); UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(settings, client, clusterService, threadPool); JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, threadPool, client, notifier); @@ -423,6 +425,7 @@ public Collection createComponents(Client client, ClusterService cluster mlLifeCycleService, jobResultsProvider, jobConfigProvider, + datafeedConfigProvider, jobManager, autodetectProcessManager, new MlInitializationService(settings, threadPool, clusterService, client), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index 229abc3843eb1..a31be5f3d7d5b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ml.action; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.FailedNodeException; @@ -23,33 +22,31 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -63,11 +60,14 @@ public class TransportCloseJobAction extends TransportTasksAction listener) { + final ClusterState state = clusterService.state(); + final DiscoveryNodes nodes = state.nodes(); + if (request.isLocal() == false && nodes.isLocalNodeElectedMaster() == false) { + // Delegates close job to elected master node, so it becomes the coordinating node. + // See comment in OpenJobAction.Transport class for more information. + if (nodes.getMasterNode() == null) { + listener.onFailure(new MasterNotDiscoveredException("no known master node")); + } else { + transportService.sendRequest(nodes.getMasterNode(), actionName, request, + new ActionListenerResponseHandler<>(listener, CloseJobAction.Response::new)); + } + } else { + /* + * Closing of multiple jobs: + * + * 1. Resolve and validate jobs first: if any job does not meet the + * criteria (e.g. open datafeed), fail immediately, do not close any + * job + * + * 2. Internally a task request is created for every open job, so there + * are n inner tasks for 1 user request + * + * 3. No task is created for closing jobs but those will be waited on + * + * 4. Collect n inner task results or failures and send 1 outer + * result/failure + */ + + PersistentTasksCustomMetaData tasksMetaData = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + expandedJobIds -> { + validate(expandedJobIds, request.isForce(), tasksMetaData, ActionListener.wrap( + response -> { + request.setOpenJobIds(response.openJobIds.toArray(new String[0])); + if (response.openJobIds.isEmpty() && response.closingJobIds.isEmpty()) { + listener.onResponse(new CloseJobAction.Response(true)); + return; + } + + if (request.isForce() == false) { + Set executorNodes = new HashSet<>(); + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); + for (String resolvedJobId : request.getOpenJobIds()) { + PersistentTasksCustomMetaData.PersistentTask jobTask = + MlTasks.getJobTask(resolvedJobId, tasks); + + if (jobTask == null || jobTask.isAssigned() == false) { + String message = "Cannot close job [" + resolvedJobId + "] because the job does not have " + + "an assigned node. Use force close to close the job"; + listener.onFailure(ExceptionsHelper.conflictStatusException(message)); + return; + } else { + executorNodes.add(jobTask.getExecutorNode()); + } + } + request.setNodes(executorNodes.toArray(new String[executorNodes.size()])); + } + + if (request.isForce()) { + List jobIdsToForceClose = new ArrayList<>(response.openJobIds); + jobIdsToForceClose.addAll(response.closingJobIds); + forceCloseJob(state, request, jobIdsToForceClose, listener); + } else { + normalCloseJob(state, task, request, response.openJobIds, response.closingJobIds, listener); + } + }, + listener::onFailure + )); + }, + listener::onFailure + )); + + } + } + + class OpenAndClosingIds { + OpenAndClosingIds() { + openJobIds = new ArrayList<>(); + closingJobIds = new ArrayList<>(); + } + List openJobIds; + List closingJobIds; } /** - * Resolve the requested jobs and add their IDs to one of the list arguments - * depending on job state. + * Separate the job Ids into open and closing job Ids and validate. + * If a job is failed it is will not be closed unless the force parameter + * in request is true. + * It is an error if the datafeed the job uses is not stopped * - * Opened jobs are added to {@code openJobIds} and closing jobs added to {@code closingJobIds}. Failed jobs are added - * to {@code openJobIds} if allowFailed is set otherwise an exception is thrown. - * @param request The close job request - * @param state Cluster state - * @param openJobIds Opened or failed jobs are added to this list - * @param closingJobIds Closing jobs are added to this list + * @param expandedJobIds The job ids + * @param forceClose Force close the job(s) + * @param tasksMetaData Persistent tasks + * @param listener Resolved job Ids listener */ - static void resolveAndValidateJobId(CloseJobAction.Request request, ClusterState state, List openJobIds, - List closingJobIds) { - PersistentTasksCustomMetaData tasksMetaData = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - final MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - - List failedJobs = new ArrayList<>(); - - Consumer jobIdProcessor = id -> { - validateJobAndTaskState(id, mlMetadata, tasksMetaData); - Job job = mlMetadata.getJobs().get(id); - if (job.isDeleting()) { - return; - } - addJobAccordingToState(id, tasksMetaData, openJobIds, closingJobIds, failedJobs); - }; - - Set expandedJobIds = mlMetadata.expandJobIds(request.getJobId(), request.allowNoJobs()); - expandedJobIds.forEach(jobIdProcessor::accept); - if (request.isForce() == false && failedJobs.size() > 0) { - if (expandedJobIds.size() == 1) { - throw ExceptionsHelper.conflictStatusException("cannot close job [{}] because it failed, use force close", - expandedJobIds.iterator().next()); - } - throw ExceptionsHelper.conflictStatusException("one or more jobs have state failed, use force close"); - } + void validate(Collection expandedJobIds, boolean forceClose, PersistentTasksCustomMetaData tasksMetaData, + ActionListener listener) { + + checkDatafeedsHaveStopped(expandedJobIds, tasksMetaData, ActionListener.wrap( + response -> { + OpenAndClosingIds ids = new OpenAndClosingIds(); + List failedJobs = new ArrayList<>(); - // allowFailed == true - openJobIds.addAll(failedJobs); + for (String jobId : expandedJobIds) { + addJobAccordingToState(jobId, tasksMetaData, ids.openJobIds, ids.closingJobIds, failedJobs); + } + + if (forceClose == false && failedJobs.size() > 0) { + if (expandedJobIds.size() == 1) { + listener.onFailure( + ExceptionsHelper.conflictStatusException("cannot close job [{}] because it failed, use force close", + expandedJobIds.iterator().next())); + return; + } + listener.onFailure( + ExceptionsHelper.conflictStatusException("one or more jobs have state failed, use force close")); + return; + } + + // If there are failed jobs force close is true + ids.openJobIds.addAll(failedJobs); + listener.onResponse(ids); + }, + listener::onFailure + )); } - private static void addJobAccordingToState(String jobId, PersistentTasksCustomMetaData tasksMetaData, + void checkDatafeedsHaveStopped(Collection jobIds, PersistentTasksCustomMetaData tasksMetaData, + ActionListener listener) { + datafeedConfigProvider.findDatafeedsForJobIds(jobIds, ActionListener.wrap( + datafeedIds -> { + for (String datafeedId : datafeedIds) { + DatafeedState datafeedState = MlTasks.getDatafeedState(datafeedId, tasksMetaData); + if (datafeedState != DatafeedState.STOPPED) { + listener.onFailure(ExceptionsHelper.conflictStatusException( + "cannot close job datafeed [{}] hasn't been stopped", datafeedId)); + return; + } + } + listener.onResponse(Boolean.TRUE); + }, + listener::onFailure + )); + } + + static void addJobAccordingToState(String jobId, PersistentTasksCustomMetaData tasksMetaData, List openJobs, List closingJobs, List failedJobs) { JobState jobState = MlTasks.getJobState(jobId, tasksMetaData); @@ -161,98 +269,6 @@ static TransportCloseJobAction.WaitForCloseRequest buildWaitForCloseRequest(List return waitForCloseRequest; } - /** - * Validate the close request. Throws an exception on any of these conditions: - *
      - *
    • If the job does not exist
      • - *
    • If the job has a data feed the feed must be closed first
      • - *
    • If the job is opening
      • - *
    - * - * @param jobId Job Id - * @param mlMetadata ML MetaData - * @param tasks Persistent tasks - */ - static void validateJobAndTaskState(String jobId, MlMetadata mlMetadata, PersistentTasksCustomMetaData tasks) { - Job job = mlMetadata.getJobs().get(jobId); - if (job == null) { - throw new ResourceNotFoundException("cannot close job, because job [" + jobId + "] does not exist"); - } - - Optional datafeed = mlMetadata.getDatafeedByJobId(jobId); - if (datafeed.isPresent()) { - DatafeedState datafeedState = MlTasks.getDatafeedState(datafeed.get().getId(), tasks); - if (datafeedState != DatafeedState.STOPPED) { - throw ExceptionsHelper.conflictStatusException("cannot close job [{}], datafeed hasn't been stopped", jobId); - } - } - } - - @Override - protected void doExecute(Task task, CloseJobAction.Request request, ActionListener listener) { - final ClusterState state = clusterService.state(); - final DiscoveryNodes nodes = state.nodes(); - if (request.isLocal() == false && nodes.isLocalNodeElectedMaster() == false) { - // Delegates close job to elected master node, so it becomes the coordinating node. - // See comment in OpenJobAction.Transport class for more information. - if (nodes.getMasterNode() == null) { - listener.onFailure(new MasterNotDiscoveredException("no known master node")); - } else { - transportService.sendRequest(nodes.getMasterNode(), actionName, request, - new ActionListenerResponseHandler<>(listener, CloseJobAction.Response::new)); - } - } else { - /* - * Closing of multiple jobs: - * - * 1. Resolve and validate jobs first: if any job does not meet the - * criteria (e.g. open datafeed), fail immediately, do not close any - * job - * - * 2. Internally a task request is created for every open job, so there - * are n inner tasks for 1 user request - * - * 3. No task is created for closing jobs but those will be waited on - * - * 4. Collect n inner task results or failures and send 1 outer - * result/failure - */ - - List openJobIds = new ArrayList<>(); - List closingJobIds = new ArrayList<>(); - resolveAndValidateJobId(request, state, openJobIds, closingJobIds); - request.setOpenJobIds(openJobIds.toArray(new String[0])); - if (openJobIds.isEmpty() && closingJobIds.isEmpty()) { - listener.onResponse(new CloseJobAction.Response(true)); - return; - } - - if (request.isForce() == false) { - Set executorNodes = new HashSet<>(); - PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); - for (String resolvedJobId : request.getOpenJobIds()) { - PersistentTasksCustomMetaData.PersistentTask jobTask = MlTasks.getJobTask(resolvedJobId, tasks); - if (jobTask == null || jobTask.isAssigned() == false) { - String message = "Cannot close job [" + resolvedJobId + "] because the job does not have an assigned node." + - " Use force close to close the job"; - listener.onFailure(ExceptionsHelper.conflictStatusException(message)); - return; - } else { - executorNodes.add(jobTask.getExecutorNode()); - } - } - request.setNodes(executorNodes.toArray(new String[executorNodes.size()])); - } - - if (request.isForce()) { - List jobIdsToForceClose = new ArrayList<>(openJobIds); - jobIdsToForceClose.addAll(closingJobIds); - forceCloseJob(state, request, jobIdsToForceClose, listener); - } else { - normalCloseJob(state, task, request, openJobIds, closingJobIds, listener); - } - } - } @Override protected void taskOperation(CloseJobAction.Request request, TransportOpenJobAction.JobTask jobTask, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java index dbde3d61d42b0..874ee8f71f5ad 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.io.IOException; +import java.util.Collections; import java.util.Map; public class TransportPutDatafeedAction extends TransportMasterNodeAction { @@ -177,7 +178,7 @@ private ElasticsearchException checkConfigsAreNotDefinedInClusterState(String da } private void checkJobDoesNotHaveADatafeed(String jobId, ActionListener listener) { - datafeedConfigProvider.findDatafeedForJobId(jobId, ActionListener.wrap( + datafeedConfigProvider.findDatafeedsForJobIds(Collections.singletonList(jobId), ActionListener.wrap( datafeedIds -> { if (datafeedIds.isEmpty()) { listener.onResponse(Boolean.TRUE); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java index 6b17721b20d68..3fd90b5d21c19 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; +import java.util.Collections; import java.util.Map; public class TransportUpdateDatafeedAction extends TransportMasterNodeAction { @@ -100,7 +101,7 @@ protected void masterOperation(UpdateDatafeedAction.Request request, ClusterStat * if it does have a datafeed it must be the one we are updating */ private void checkJobDoesNotHaveADifferentDatafeed(String jobId, String datafeedId, ActionListener listener) { - datafeedConfigProvider.findDatafeedForJobId(jobId, ActionListener.wrap( + datafeedConfigProvider.findDatafeedsForJobIds(Collections.singletonList(jobId), ActionListener.wrap( datafeedIds -> { if (datafeedIds.isEmpty()) { // Ok the job does not have a datafeed diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index 22912e9afc795..758c190feef1b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -56,6 +56,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -168,17 +169,17 @@ public void onFailure(Exception e) { } /** - * Find any datafeeds that are used by job {@code jobid} i.e. the - * datafeed that references job {@code jobid}. + * Find any datafeeds that are used by jobs {@code jobIds} i.e. the + * datafeeds that references any of the jobs in {@code jobIds}. * * In theory there should never be more than one datafeed referencing a * particular job. * - * @param jobId The job to find + * @param jobIds The jobs to find the datafeeds of * @param listener Datafeed Id listener */ - public void findDatafeedForJobId(String jobId, ActionListener> listener) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdQuery(jobId)); + public void findDatafeedsForJobIds(Collection jobIds, ActionListener> listener) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdsQuery(jobIds)); sourceBuilder.fetchSource(false); sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName()); @@ -191,8 +192,8 @@ public void findDatafeedForJobId(String jobId, ActionListener> liste response -> { Set datafeedIds = new HashSet<>(); SearchHit[] hits = response.getHits().getHits(); - // There should be 0 or 1 datafeeds referencing the same job - assert hits.length <= 1; + // There cannot be more than one datafeed per job + assert hits.length <= jobIds.size(); for (SearchHit hit : hits) { datafeedIds.add(hit.field(DatafeedConfig.ID.getPreferredName()).getValue()); @@ -463,10 +464,10 @@ private QueryBuilder buildDatafeedIdQuery(String [] tokens) { return boolQueryBuilder; } - private QueryBuilder buildDatafeedJobIdQuery(String jobId) { + private QueryBuilder buildDatafeedJobIdsQuery(Collection jobIds) { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.filter(new TermQueryBuilder(DatafeedConfig.CONFIG_TYPE.getPreferredName(), DatafeedConfig.TYPE)); - boolQueryBuilder.filter(new TermQueryBuilder(Job.ID.getPreferredName(), jobId)); + boolQueryBuilder.filter(new TermsQueryBuilder(Job.ID.getPreferredName(), jobIds)); return boolQueryBuilder; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 879e4c3fa18af..81dfa38148fad 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -6,15 +6,18 @@ package org.elasticsearch.xpack.ml.action; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -27,225 +30,179 @@ import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobState; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; -import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; +import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests.addJobTask; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TransportCloseJobActionTests extends ESTestCase { - public void testValidate_datafeedIsStarted() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id").build(new Date()), false); - mlBuilder.putDatafeed(BaseMlIntegTestCase.createDatafeed("datafeed_id", "job_id", - Collections.singletonList("*")), Collections.emptyMap()); - final PersistentTasksCustomMetaData.Builder startDataFeedTaskBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id", null, JobState.OPENED, startDataFeedTaskBuilder); - addTask("datafeed_id", 0L, null, DatafeedState.STARTED, startDataFeedTaskBuilder); - - ElasticsearchStatusException e = - expectThrows(ElasticsearchStatusException.class, - () -> TransportCloseJobAction.validateJobAndTaskState("job_id", mlBuilder.build(), - startDataFeedTaskBuilder.build())); - assertEquals(RestStatus.CONFLICT, e.status()); - assertEquals("cannot close job [job_id], datafeed hasn't been stopped", e.getMessage()); + private ClusterService clusterService; + private JobConfigProvider jobConfigProvider; + private DatafeedConfigProvider datafeedConfigProvider; - final PersistentTasksCustomMetaData.Builder dataFeedNotStartedTaskBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id", null, JobState.OPENED, dataFeedNotStartedTaskBuilder); - if (randomBoolean()) { - addTask("datafeed_id", 0L, null, DatafeedState.STOPPED, dataFeedNotStartedTaskBuilder); - } - - TransportCloseJobAction.validateJobAndTaskState("job_id", mlBuilder.build(), dataFeedNotStartedTaskBuilder.build()); + @Before + private void setupMocks() { + clusterService = mock(ClusterService.class); + jobConfigProvider = mock(JobConfigProvider.class); + datafeedConfigProvider = mock(DatafeedConfigProvider.class); } - public void testValidate_jobIsOpening() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("opening-job").build(new Date()), false); - - // An opening job has a null status field - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("opening-job", null, null, tasksBuilder); + public void testAddJobAccordingToState() { + List openJobIds = new ArrayList<>(); + List closingJobIds = new ArrayList<>(); + List failedJobIds = new ArrayList<>(); - TransportCloseJobAction.validateJobAndTaskState("opening-job", mlBuilder.build(), tasksBuilder.build()); - } + PersistentTasksCustomMetaData.Builder taskBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask("open-job", null, JobState.OPENED, taskBuilder); + addJobTask("failed-job", null, JobState.FAILED, taskBuilder); + addJobTask("closing-job", null, JobState.CLOSING, taskBuilder); + addJobTask("opening-job", null, JobState.OPENING, taskBuilder); + PersistentTasksCustomMetaData tasks = taskBuilder.build(); - public void testValidate_jobIsMissing() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("missing-job", null, null, tasksBuilder); - - expectThrows(ResourceNotFoundException.class, () -> - TransportCloseJobAction.validateJobAndTaskState("missing-job", mlBuilder.build(), tasksBuilder.build())); - } - - public void testResolve_givenAll() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_1").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_2").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_3").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_4").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_5").build(new Date()), false); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id_1", null, JobState.OPENED, tasksBuilder); - addJobTask("job_id_2", null, JobState.OPENED, tasksBuilder); - addJobTask("job_id_3", null, JobState.FAILED, tasksBuilder); - addJobTask("job_id_4", null, JobState.CLOSING, tasksBuilder); - - ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) - .build(); - - List openJobs = new ArrayList<>(); - List closingJobs = new ArrayList<>(); - - CloseJobAction.Request request = new CloseJobAction.Request("_all"); - request.setForce(true); - TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs); - assertEquals(Arrays.asList("job_id_1", "job_id_2", "job_id_3"), openJobs); - assertEquals(Collections.singletonList("job_id_4"), closingJobs); - - request.setForce(false); - expectThrows(ElasticsearchStatusException.class, - () -> TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs)); + for (String id : new String [] {"open-job", "closing-job", "opening-job", "failed-job"}) { + TransportCloseJobAction.addJobAccordingToState(id, tasks, openJobIds, closingJobIds, failedJobIds); + } + assertThat(openJobIds, containsInAnyOrder("open-job", "opening-job")); + assertThat(failedJobIds, contains("failed-job")); + assertThat(closingJobIds, contains("closing-job")); } - public void testResolve_givenJobId() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_1").build(new Date()), false); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id_1", null, JobState.OPENED, tasksBuilder); + public void testValidate_datafeedState() { + final PersistentTasksCustomMetaData.Builder startDataFeedTaskBuilder = PersistentTasksCustomMetaData.builder(); + String jobId = "job-with-started-df"; + String datafeedId = "df1"; + addJobTask(jobId, null, JobState.OPENED, startDataFeedTaskBuilder); + addTask(datafeedId, 0L, null, DatafeedState.STARTED, startDataFeedTaskBuilder); - ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) - .build(); + mockDatafeedConfigFindDatafeeds(Collections.singleton(datafeedId)); - List openJobs = new ArrayList<>(); - List closingJobs = new ArrayList<>(); + TransportCloseJobAction closeJobAction = createAction(); - CloseJobAction.Request request = new CloseJobAction.Request("job_id_1"); - TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs); - assertEquals(Collections.singletonList("job_id_1"), openJobs); - assertEquals(Collections.emptyList(), closingJobs); + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + responseHolder::set, + exceptionHolder::set + ); - // Job without task is closed - cs1 = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())) - .build(); + closeJobAction.validate(Arrays.asList(jobId), false, startDataFeedTaskBuilder.build(), listener); - openJobs.clear(); - closingJobs.clear(); - TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs); - assertEquals(Collections.emptyList(), openJobs); - assertEquals(Collections.emptyList(), closingJobs); - } - - public void testResolve_throwsWithUnknownJobId() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_1").build(new Date()), false); - - ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())) - .build(); + assertNull(responseHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ElasticsearchStatusException.class)); + ElasticsearchStatusException esException = (ElasticsearchStatusException) exceptionHolder.get(); + assertEquals(RestStatus.CONFLICT, esException.status()); + assertEquals("cannot close job datafeed [df1] hasn't been stopped", esException.getMessage()); - List openJobs = new ArrayList<>(); - List closingJobs = new ArrayList<>(); + final PersistentTasksCustomMetaData.Builder dataFeedNotStartedTaskBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask(jobId, null, JobState.OPENED, dataFeedNotStartedTaskBuilder); + if (randomBoolean()) { + addTask(datafeedId, 0L, null, DatafeedState.STOPPED, dataFeedNotStartedTaskBuilder); + } - CloseJobAction.Request request = new CloseJobAction.Request("missing-job"); - expectThrows(ResourceNotFoundException.class, - () -> TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs)); + exceptionHolder.set(null); + closeJobAction.validate(Arrays.asList(jobId), false, dataFeedNotStartedTaskBuilder.build(), listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertThat(responseHolder.get().openJobIds, contains(jobId)); + assertThat(responseHolder.get().closingJobIds, empty()); } - public void testResolve_givenJobIdFailed() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_failed").build(new Date()), false); - + public void testValidate_givenFailedJob() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id_failed", null, JobState.FAILED, tasksBuilder); - ClusterState cs1 = ClusterState.builder(new ClusterName("_name")).metaData(new MetaData.Builder() - .putCustom(MlMetadata.TYPE, mlBuilder.build()).putCustom(PersistentTasksCustomMetaData.TYPE, - tasksBuilder.build())).build(); - - List openJobs = new ArrayList<>(); - List closingJobs = new ArrayList<>(); - - CloseJobAction.Request request = new CloseJobAction.Request("job_id_failed"); - request.setForce(true); - - TransportCloseJobAction.resolveAndValidateJobId(request, cs1, openJobs, closingJobs); - assertEquals(Collections.singletonList("job_id_failed"), openJobs); - assertEquals(Collections.emptyList(), closingJobs); - - openJobs.clear(); - closingJobs.clear(); - - request.setForce(false); - expectThrows(ElasticsearchStatusException.class, () -> TransportCloseJobAction.resolveAndValidateJobId(request, cs1, - openJobs, closingJobs)); + mockDatafeedConfigFindDatafeeds(Collections.emptySet()); + + TransportCloseJobAction closeJobAction = createAction(); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + responseHolder::set, + exceptionHolder::set + ); + + // force close so not an error for the failed job + closeJobAction.validate(Arrays.asList("job_id_failed"), true, tasksBuilder.build(), listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertThat(responseHolder.get().openJobIds, contains("job_id_failed")); + assertThat(responseHolder.get().closingJobIds, empty()); + + // not a force close so is an error + responseHolder.set(null); + closeJobAction.validate(Arrays.asList("job_id_failed"), false, tasksBuilder.build(), listener); + assertNull(responseHolder.get()); + assertNotNull(exceptionHolder.get()); + assertThat(exceptionHolder.get(), instanceOf(ElasticsearchStatusException.class)); + ElasticsearchStatusException esException = (ElasticsearchStatusException) exceptionHolder.get(); + assertEquals(RestStatus.CONFLICT, esException.status()); + assertEquals("cannot close job [job_id_failed] because it failed, use force close", esException.getMessage()); } - public void testResolve_withSpecificJobIds() { - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_closing").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_open-1").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_open-2").build(new Date()), false); - mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_closed").build(new Date()), false); - + public void testValidate_withSpecificJobIds() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id_closing", null, JobState.CLOSING, tasksBuilder); addJobTask("job_id_open-1", null, JobState.OPENED, tasksBuilder); addJobTask("job_id_open-2", null, JobState.OPENED, tasksBuilder); - // closed job has no task - - ClusterState cs1 = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) - .build(); - - List openJobs = new ArrayList<>(); - List closingJobs = new ArrayList<>(); - - TransportCloseJobAction.resolveAndValidateJobId(new CloseJobAction.Request("_all"), cs1, openJobs, closingJobs); - assertEquals(Arrays.asList("job_id_open-1", "job_id_open-2"), openJobs); - assertEquals(Collections.singletonList("job_id_closing"), closingJobs); - openJobs.clear(); - closingJobs.clear(); - - TransportCloseJobAction.resolveAndValidateJobId(new CloseJobAction.Request("*open*"), cs1, openJobs, closingJobs); - assertEquals(Arrays.asList("job_id_open-1", "job_id_open-2"), openJobs); - assertEquals(Collections.emptyList(), closingJobs); - openJobs.clear(); - closingJobs.clear(); - - TransportCloseJobAction.resolveAndValidateJobId(new CloseJobAction.Request("job_id_closing"), cs1, openJobs, closingJobs); - assertEquals(Collections.emptyList(), openJobs); - assertEquals(Collections.singletonList("job_id_closing"), closingJobs); - openJobs.clear(); - closingJobs.clear(); - - TransportCloseJobAction.resolveAndValidateJobId(new CloseJobAction.Request("job_id_open-1"), cs1, openJobs, closingJobs); - assertEquals(Collections.singletonList("job_id_open-1"), openJobs); - assertEquals(Collections.emptyList(), closingJobs); - openJobs.clear(); - closingJobs.clear(); + PersistentTasksCustomMetaData tasks = tasksBuilder.build(); + + mockDatafeedConfigFindDatafeeds(Collections.emptySet()); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + responseHolder::set, + exceptionHolder::set + ); + + TransportCloseJobAction closeJobAction = createAction(); + closeJobAction.validate(Arrays.asList("job_id_closing", "job_id_open-1", "job_id_open-2"), false, tasks, listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertEquals(Arrays.asList("job_id_open-1", "job_id_open-2"), responseHolder.get().openJobIds); + assertEquals(Collections.singletonList("job_id_closing"), responseHolder.get().closingJobIds); + + closeJobAction.validate(Arrays.asList("job_id_open-1", "job_id_open-2"), false, tasks, listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertEquals(Arrays.asList("job_id_open-1", "job_id_open-2"), responseHolder.get().openJobIds); + assertEquals(Collections.emptyList(), responseHolder.get().closingJobIds); + + closeJobAction.validate(Arrays.asList("job_id_closing"), false, tasks, listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertEquals(Collections.emptyList(), responseHolder.get().openJobIds); + assertEquals(Arrays.asList("job_id_closing"), responseHolder.get().closingJobIds); + + closeJobAction.validate(Arrays.asList("job_id_open-1"), false, tasks, listener); + assertNull(exceptionHolder.get()); + assertNotNull(responseHolder.get()); + assertEquals(Arrays.asList("job_id_open-1"), responseHolder.get().openJobIds); + assertEquals(Collections.emptyList(), responseHolder.get().closingJobIds); } public void testDoExecute_whenNothingToClose() { @@ -256,16 +213,13 @@ public void testDoExecute_whenNothingToClose() { addJobTask("foo", null, JobState.CLOSED, tasksBuilder); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) .build(); - ClusterService clusterService = mock(ClusterService.class); + TransportCloseJobAction transportAction = createAction(); when(clusterService.state()).thenReturn(clusterState); - - TransportCloseJobAction transportAction = new TransportCloseJobAction(Settings.EMPTY, - mock(TransportService.class), mock(ThreadPool.class), mock(ActionFilters.class), - clusterService, mock(Client.class), mock(Auditor.class), mock(PersistentTasksService.class)); + mockJobConfigProviderExpandIds(Collections.singleton("foo")); + mockDatafeedConfigFindDatafeeds(Collections.emptySet()); AtomicBoolean gotResponse = new AtomicBoolean(false); CloseJobAction.Request request = new Request("foo"); @@ -316,4 +270,29 @@ public static void addTask(String datafeedId, long startTime, String nodeId, Dat tasks.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state); } + private TransportCloseJobAction createAction() { + return new TransportCloseJobAction(Settings.EMPTY, + mock(TransportService.class), mock(ThreadPool.class), mock(ActionFilters.class), + clusterService, mock(Client.class), mock(Auditor.class), mock(PersistentTasksService.class), + jobConfigProvider, datafeedConfigProvider); + } + + private void mockDatafeedConfigFindDatafeeds(Set datafeedIds) { + doAnswer(invocation -> { + ActionListener> listener = (ActionListener>) invocation.getArguments()[1]; + listener.onResponse(datafeedIds); + + return null; + }).when(datafeedConfigProvider).findDatafeedsForJobIds(any(), any(ActionListener.class)); + } + + private void mockJobConfigProviderExpandIds(Set expandedIds) { + doAnswer(invocation -> { + ActionListener> listener = (ActionListener>) invocation.getArguments()[2]; + listener.onResponse(expandedIds); + + return null; + }).when(jobConfigProvider).expandJobsIds(any(), anyBoolean(), any(ActionListener.class)); + } + } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index c4dd8c19ea611..8acee83e0b0b6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -289,7 +289,7 @@ public void testExpandDatafeeds() throws Exception { assertThat(expandedDatafeeds, containsInAnyOrder(bar1, foo1, foo2)); } - public void testFindDatafeedForJobId() throws Exception { + public void testFindDatafeedsForJobIds() throws Exception { putDatafeedConfig(createDatafeedConfig("foo-1", "j1"), Collections.emptyMap()); putDatafeedConfig(createDatafeedConfig("foo-2", "j2"), Collections.emptyMap()); putDatafeedConfig(createDatafeedConfig("bar-1", "j3"), Collections.emptyMap()); @@ -299,17 +299,17 @@ public void testFindDatafeedForJobId() throws Exception { AtomicReference> datafeedIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("new-job", actionListener), + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedsForJobIds(Collections.singletonList("new-job"), actionListener), datafeedIdsHolder, exceptionHolder); assertThat(datafeedIdsHolder.get(), empty()); - blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("j2", actionListener), + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedsForJobIds(Collections.singletonList("j2"), actionListener), datafeedIdsHolder, exceptionHolder); assertThat(datafeedIdsHolder.get(), contains("foo-2")); - blockingCall(actionListener -> datafeedConfigProvider.findDatafeedForJobId("j3", actionListener), + blockingCall(actionListener -> datafeedConfigProvider.findDatafeedsForJobIds(Arrays.asList("j3", "j1"), actionListener), datafeedIdsHolder, exceptionHolder); - assertThat(datafeedIdsHolder.get(), contains("bar-1")); + assertThat(datafeedIdsHolder.get(), containsInAnyOrder("bar-1", "foo-1")); } public void testHeadersAreOverwritten() throws Exception { From e6e78fe09ede39d47d8b0748344234c619b43d84 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 16 Oct 2018 11:46:56 +0100 Subject: [PATCH 09/57] [ML] Adjust finalize job action to work with documents (#34226) --- .../elasticsearch/xpack/core/ml/MlTasks.java | 2 +- .../xpack/core/ml/action/UpdateJobAction.java | 21 ++--- .../xpack/core/ml/job/config/JobUpdate.java | 4 +- .../action/UpdateJobActionRequestTests.java | 10 ++- .../core/ml/job/config/JobUpdateTests.java | 2 +- .../ml/action/TransportCloseJobAction.java | 9 +- .../TransportFinalizeJobExecutionAction.java | 46 +--------- .../ml/action/TransportOpenJobAction.java | 2 +- .../ml/job/persistence/JobConfigProvider.java | 5 +- .../job/persistence/JobResultsPersister.java | 4 +- .../output/AutoDetectResultProcessor.java | 86 ++++++++++++------- .../action/TransportCloseJobActionTests.java | 2 +- .../ml/job/persistence/MockClientBuilder.java | 2 +- .../AutoDetectResultProcessorTests.java | 23 +++-- 14 files changed, 100 insertions(+), 118 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index a56d3d639239d..46685001153d7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -97,7 +97,7 @@ public static Set openJobIds(PersistentTasksCustomMetaData tasks) { * Is there an ml anomaly detector job task for the job {@code jobId}? * @param jobId The job id * @param tasks Persistent tasks - * @return + * @return True if the job has a task */ public static boolean taskExistsForJob(String jobId, PersistentTasksCustomMetaData tasks) { return openJobIds(tasks).contains(jobId); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java index d4fe804c451af..f7e2e514e5769 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java @@ -49,7 +49,6 @@ public static UpdateJobAction.Request parseRequest(String jobId, XContentParser /** Indicates an update that was not triggered by a user */ private boolean isInternal; - private boolean waitForAck = true; public Request(String jobId, JobUpdate update) { this(jobId, update, false); @@ -83,14 +82,6 @@ public boolean isInternal() { return isInternal; } - public boolean isWaitForAck() { - return waitForAck; - } - - public void setWaitForAck(boolean waitForAck) { - this.waitForAck = waitForAck; - } - @Override public ActionRequestValidationException validate() { return null; @@ -106,10 +97,9 @@ public void readFrom(StreamInput in) throws IOException { } else { isInternal = false; } - if (in.getVersion().onOrAfter(Version.V_6_3_0)) { - waitForAck = in.readBoolean(); - } else { - waitForAck = true; + // TODO jindex change CURRENT to specific version when feature branch is merged + if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.getVersion().before(Version.CURRENT)) { + in.readBoolean(); // was waitForAck } } @@ -121,8 +111,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_2_2)) { out.writeBoolean(isInternal); } - if (out.getVersion().onOrAfter(Version.V_6_3_0)) { - out.writeBoolean(waitForAck); + // TODO jindex change CURRENT to specific version when feature branch is merged + if (out.getVersion().onOrAfter(Version.V_6_3_0) && out.getVersion().before(Version.CURRENT)) { + out.writeBoolean(false); // was waitForAck } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 37fbf3dab14d8..44e20846f9aa3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -60,7 +60,7 @@ public class JobUpdate implements Writeable, ToXContentObject { INTERNAL_PARSER.declareLong(Builder::setEstablishedModelMemory, Job.ESTABLISHED_MODEL_MEMORY); INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION); - INTERNAL_PARSER.declareBoolean(Builder::setClearJobFinishTime, CLEAR_JOB_FINISH_TIME); + INTERNAL_PARSER.declareBoolean(Builder::setClearFinishTime, CLEAR_JOB_FINISH_TIME); } private final String jobId; @@ -753,7 +753,7 @@ public Builder setJobVersion(String version) { return this; } - public Builder setClearJobFinishTime(boolean clearJobFinishTime) { + public Builder setClearFinishTime(boolean clearJobFinishTime) { this.clearJobFinishTime = clearJobFinishTime; return this; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java index 3b09017147886..20d27f03d0c29 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java @@ -18,8 +18,14 @@ protected UpdateJobAction.Request createTestInstance() { // no need to randomize JobUpdate this is already tested in: JobUpdateTests JobUpdate.Builder jobUpdate = new JobUpdate.Builder(jobId); jobUpdate.setAnalysisLimits(new AnalysisLimits(100L, 100L)); - UpdateJobAction.Request request = new UpdateJobAction.Request(jobId, jobUpdate.build()); - request.setWaitForAck(randomBoolean()); + UpdateJobAction.Request request; + if (randomBoolean()) { + request = new UpdateJobAction.Request(jobId, jobUpdate.build()); + } else { + // this call sets isInternal = true + request = UpdateJobAction.Request.internal(jobId, jobUpdate.build()); + } + return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 35f72ced1e13b..c6eb42038901b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -97,7 +97,7 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { update.setJobVersion(randomFrom(Version.CURRENT, Version.V_6_2_0, Version.V_6_1_0)); } if (useInternalParser) { - update.setClearJobFinishTime(randomBoolean()); + update.setClearFinishTime(randomBoolean()); } return update.build(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index a31be5f3d7d5b..e6cc8b69e1bd1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; @@ -49,9 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; - public class TransportCloseJobAction extends TransportTasksAction { @@ -427,10 +423,7 @@ void waitForJobClosed(CloseJobAction.Request request, WaitForCloseRequest waitFo }, request.getCloseTimeout(), new ActionListener() { @Override public void onResponse(Boolean result) { - FinalizeJobExecutionAction.Request finalizeRequest = new FinalizeJobExecutionAction.Request( - waitForCloseRequest.jobsToFinalize.toArray(new String[0])); - executeAsyncWithOrigin(client, ML_ORIGIN, FinalizeJobExecutionAction.INSTANCE, finalizeRequest, - ActionListener.wrap(r -> listener.onResponse(response), listener::onFailure)); + listener.onResponse(response); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java index fb56e61983973..c9fdd7b18fb53 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java @@ -10,22 +10,15 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; -import org.elasticsearch.xpack.core.ml.job.config.Job; - -import java.util.Date; public class TransportFinalizeJobExecutionAction extends TransportMasterNodeAction { @@ -51,41 +44,10 @@ protected AcknowledgedResponse newResponse() { @Override protected void masterOperation(FinalizeJobExecutionAction.Request request, ClusterState state, - ActionListener listener) throws Exception { - String jobIdString = String.join(",", request.getJobIds()); - String source = "finalize_job_execution [" + jobIdString + "]"; - logger.debug("finalizing jobs [{}]", jobIdString); - clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - XPackPlugin.checkReadyForXPackCustomMetadata(currentState); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(currentState); - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(mlMetadata); - Date finishedTime = new Date(); - - for (String jobId : request.getJobIds()) { - Job.Builder jobBuilder = new Job.Builder(mlMetadata.getJobs().get(jobId)); - jobBuilder.setFinishedTime(finishedTime); - mlMetadataBuilder.putJob(jobBuilder.build(), true); - } - ClusterState.Builder builder = ClusterState.builder(currentState); - return builder.metaData(new MetaData.Builder(currentState.metaData()) - .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build())) - .build(); - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, - ClusterState newState) { - logger.debug("finalized job [{}]", jobIdString); - listener.onResponse(new AcknowledgedResponse(true)); - } - }); + ActionListener listener) { + // This action is no longer required but needs to be preserved + // in case it is called by an old node in a mixed cluster + listener.onResponse(new AcknowledgedResponse(true)); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index ebabcab59d674..daa1c789f12ab 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -563,7 +563,7 @@ public void onTimeout(TimeValue timeout) { } private void clearJobFinishedTime(String jobId, ActionListener listener) { - JobUpdate update = new JobUpdate.Builder(jobId).setClearJobFinishTime(true).build(); + JobUpdate update = new JobUpdate.Builder(jobId).setClearFinishTime(true).build(); jobConfigProvider.updateJob(jobId, update, null, ActionListener.wrap( job -> listener.onResponse(new AcknowledgedResponse(true)), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index feab1e84a0146..26e9ee3019b04 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -237,7 +237,9 @@ public void onFailure(Exception e) { * * @param jobId The Id of the job to update * @param update The job update - * @param maxModelMemoryLimit The maximum model memory allowed + * @param maxModelMemoryLimit The maximum model memory allowed. This can be {@code null} + * if the job's {@link org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits} + * are not changed. * @param updatedJobListener Updated job listener */ public void updateJob(String jobId, JobUpdate update, ByteSizeValue maxModelMemoryLimit, ActionListener updatedJobListener) { @@ -373,7 +375,6 @@ private void indexUpdatedJob(Job updatedJob, long version, ActionListener u } } - /** * Check a job exists. A job exists if it has a configuration document. * If the .ml-config index does not exist it is treated as a missing job diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java index 233a2b4078ac7..9efdbc1975716 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java @@ -242,10 +242,10 @@ public void persistQuantiles(Quantiles quantiles, WriteRequest.RefreshPolicy ref /** * Persist a model snapshot description */ - public void persistModelSnapshot(ModelSnapshot modelSnapshot, WriteRequest.RefreshPolicy refreshPolicy) { + public IndexResponse persistModelSnapshot(ModelSnapshot modelSnapshot, WriteRequest.RefreshPolicy refreshPolicy) { Persistable persistable = new Persistable(modelSnapshot.getJobId(), modelSnapshot, ModelSnapshot.documentId(modelSnapshot)); persistable.setRefreshPolicy(refreshPolicy); - persistable.persist(AnomalyDetectorsIndex.resultsWriteAlias(modelSnapshot.getJobId())).actionGet(); + return persistable.persist(AnomalyDetectorsIndex.resultsWriteAlias(modelSnapshot.getJobId())).actionGet(); } /** diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java index b86ec4de8257f..e07e6d6bc7e6d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -8,7 +8,12 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.Loggers; @@ -18,10 +23,10 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.xpack.core.ml.MachineLearningField; -import org.elasticsearch.xpack.core.ml.action.PutJobAction; -import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; -import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; @@ -34,17 +39,20 @@ import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; import org.elasticsearch.xpack.ml.notifications.Auditor; import java.time.Duration; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; @@ -164,7 +172,7 @@ public void process(AutodetectProcess process) { } LOGGER.info("[{}] {} buckets parsed from autodetect output", jobId, bucketCount); - runEstablishedModelMemoryUpdate(true); + onAutodetectClose(); } catch (Exception e) { failed = true; @@ -269,8 +277,10 @@ void processResult(Context context, AutodetectResult result) { ModelSnapshot modelSnapshot = result.getModelSnapshot(); if (modelSnapshot != null) { // We need to refresh in order for the snapshot to be available when we try to update the job with it - persister.persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE); - updateModelSnapshotOnJob(modelSnapshot); + IndexResponse indexResponse = persister.persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE); + if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) { + updateModelSnapshotOnJob(modelSnapshot); + } } Quantiles quantiles = result.getQuantiles(); if (quantiles != null) { @@ -334,12 +344,6 @@ private void notifyModelMemoryStatusChange(Context context, ModelSizeStats model } protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { - JobUpdate update = new JobUpdate.Builder(jobId) - .setModelSnapshotId(modelSnapshot.getSnapshotId()) - .setModelSnapshotMinVersion(modelSnapshot.getMinVersion()) - .build(); - UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(jobId, update); - try { // This blocks the main processing thread in the unlikely event // there are 2 model snapshots queued up. But it also has the @@ -351,20 +355,25 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { return; } - executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest, new ActionListener() { - @Override - public void onResponse(PutJobAction.Response response) { - updateModelSnapshotSemaphore.release(); - LOGGER.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId()); - } + Map update = new HashMap<>(); + update.put(Job.MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshot.getSnapshotId()); + update.put(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshot.getMinVersion().toString()); - @Override - public void onFailure(Exception e) { - updateModelSnapshotSemaphore.release(); - LOGGER.error("[" + jobId + "] Failed to update job with new model snapshot id [" + - modelSnapshot.getSnapshotId() + "]", e); - } - }); + updateJob(jobId, Collections.singletonMap(Job.MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshot.getSnapshotId()), + new ActionListener() { + @Override + public void onResponse(UpdateResponse updateResponse) { + updateModelSnapshotSemaphore.release(); + LOGGER.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId()); + } + + @Override + public void onFailure(Exception e) { + updateModelSnapshotSemaphore.release(); + LOGGER.error("[" + jobId + "] Failed to update job with new model snapshot id [" + + modelSnapshot.getSnapshotId() + "]", e); + } + }); } /** @@ -422,6 +431,13 @@ private synchronized void runEstablishedModelMemoryUpdate(boolean cancelExisting } } + private void onAutodetectClose() { + updateJob(jobId, Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date()), ActionListener.wrap( + r -> runEstablishedModelMemoryUpdate(true), + e -> LOGGER.error("[" + jobId + "] Failed to finalize job on autodetect close", e)) + ); + } + private void updateEstablishedModelMemoryOnJob() { // Copy these before committing writes, so the calculation is done based on committed documents @@ -433,14 +449,10 @@ private void updateEstablishedModelMemoryOnJob() { jobResultsProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, modelSizeStatsForCalc, establishedModelMemory -> { if (latestEstablishedModelMemory != establishedModelMemory) { - JobUpdate update = new JobUpdate.Builder(jobId).setEstablishedModelMemory(establishedModelMemory).build(); - UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(jobId, update); - updateRequest.setWaitForAck(false); - - executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest, - new ActionListener() { + updateJob(jobId, Collections.singletonMap(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory), + new ActionListener() { @Override - public void onResponse(PutJobAction.Response response) { + public void onResponse(UpdateResponse response) { latestEstablishedModelMemory = establishedModelMemory; LOGGER.debug("[{}] Updated job with established model memory [{}]", jobId, establishedModelMemory); } @@ -455,6 +467,14 @@ public void onFailure(Exception e) { }, e -> LOGGER.error("[" + jobId + "] Failed to calculate established model memory", e)); } + private void updateJob(String jobId, Map update, ActionListener listener) { + UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + updateRequest.retryOnConflict(3); + updateRequest.doc(update); + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateAction.INSTANCE, updateRequest, listener); + } + public void awaitCompletion() throws TimeoutException { try { // Although the results won't take 30 minutes to finish, the pipe won't be closed diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 81dfa38148fad..b7d73b79829ce 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -272,7 +272,7 @@ public static void addTask(String datafeedId, long startTime, String nodeId, Dat private TransportCloseJobAction createAction() { return new TransportCloseJobAction(Settings.EMPTY, - mock(TransportService.class), mock(ThreadPool.class), mock(ActionFilters.class), + mock(TransportService.class), mock(ThreadPool.class), mock(ActionFilters.class), clusterService, mock(Client.class), mock(Auditor.class), mock(PersistentTasksService.class), jobConfigProvider, datafeedConfigProvider); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 726b815728f52..c10af20aba79f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -274,7 +274,7 @@ public MockClientBuilder prepareSearch(String index, String type, int from, int * Creates a {@link SearchResponse} with a {@link SearchHit} for each element of {@code docs} * @param indexName Index being searched * @param docs Returned in the SearchResponse - * @return + * @return this */ @SuppressWarnings("unchecked") public MockClientBuilder prepareSearch(String indexName, List docs) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java index 05b6bc7209b87..37bd367c28e2c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java @@ -8,17 +8,20 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; -import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -29,14 +32,15 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.After; import org.junit.Before; +import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import java.time.Duration; @@ -50,6 +54,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; @@ -67,7 +72,7 @@ public class AutoDetectResultProcessorTests extends ESTestCase { - private static final String JOB_ID = "_id"; + private static final String JOB_ID = "valid_id"; private static final long BUCKET_SPAN_MS = 1000; private ThreadPool threadPool; @@ -90,6 +95,8 @@ public void setUpMocks() { when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); renormalizer = mock(Renormalizer.class); persister = mock(JobResultsPersister.class); + when(persister.persistModelSnapshot(any(), any())) + .thenReturn(new IndexResponse(new ShardId("ml", "uid", 0), "doc", "1", 0L, 0L, 0L, true)); jobResultsProvider = mock(JobResultsProvider.class); flushListener = mock(FlushListener.class); processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister, jobResultsProvider, @@ -411,11 +418,13 @@ public void testProcessResult_modelSnapshot() { processorUnderTest.processResult(context, result); verify(persister, times(1)).persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE); - UpdateJobAction.Request expectedJobUpdateRequest = UpdateJobAction.Request.internal(JOB_ID, - new JobUpdate.Builder(JOB_ID).setModelSnapshotId("a_snapshot_id").setModelSnapshotMinVersion(Version.CURRENT).build()); - verify(client).execute(same(UpdateJobAction.INSTANCE), eq(expectedJobUpdateRequest), any()); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(UpdateRequest.class); + verify(client).execute(same(UpdateAction.INSTANCE), requestCaptor.capture(), any()); verifyNoMoreInteractions(persister); + + UpdateRequest capturedRequest = requestCaptor.getValue(); + assertThat(capturedRequest.doc().sourceAsMap().keySet(), contains(Job.MODEL_SNAPSHOT_ID.getPreferredName())); } public void testProcessResult_quantiles_givenRenormalizationIsEnabled() { From f8614a10d8ee74f4d39758cb2ceb129075ffdef9 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 17 Oct 2018 13:49:59 +0100 Subject: [PATCH 10/57] [ML] Job in index: Datafeed node selector (#34218) --- .../core/ml/action/StartDatafeedAction.java | 46 ++++- .../core/ml/action/DatafeedParamsTests.java | 8 + .../xpack/ml/MachineLearning.java | 3 +- .../xpack/ml/MlAssignmentNotifier.java | 24 +-- .../action/TransportStartDatafeedAction.java | 166 +++++++++++------- .../xpack/ml/datafeed/DatafeedJob.java | 4 + .../xpack/ml/datafeed/DatafeedJobBuilder.java | 115 +++++++++--- .../xpack/ml/datafeed/DatafeedManager.java | 68 ++++--- .../ml/datafeed/DatafeedNodeSelector.java | 38 ++-- .../xpack/ml/MlAssignmentNotifierTests.java | 4 +- .../TransportStartDatafeedActionTests.java | 36 +--- .../ml/datafeed/DatafeedJobBuilderTests.java | 66 +++++-- .../ml/datafeed/DatafeedManagerTests.java | 43 ++--- .../datafeed/DatafeedNodeSelectorTests.java | 127 ++++++-------- 14 files changed, 437 insertions(+), 311 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index a8841578f28c7..c0f7b1a1229d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -28,10 +28,13 @@ import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.function.LongSupplier; @@ -141,6 +144,8 @@ public boolean equals(Object obj) { public static class DatafeedParams implements XPackPlugin.XPackPersistentTaskParams { + public static final ParseField INDICES = new ParseField("indices"); + public static ObjectParser PARSER = new ObjectParser<>(MlTasks.DATAFEED_TASK_NAME, true, DatafeedParams::new); static { PARSER.declareString((params, datafeedId) -> params.datafeedId = datafeedId, DatafeedConfig.ID); @@ -149,6 +154,8 @@ public static class DatafeedParams implements XPackPlugin.XPackPersistentTaskPar PARSER.declareString(DatafeedParams::setEndTime, END_TIME); PARSER.declareString((params, val) -> params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); + PARSER.declareString(DatafeedParams::setJobId, Job.ID); + PARSER.declareStringArray(DatafeedParams::setDatafeedIndices, INDICES); } static long parseDateOrThrow(String date, ParseField paramName, LongSupplier now) { @@ -188,6 +195,10 @@ public DatafeedParams(StreamInput in) throws IOException { startTime = in.readVLong(); endTime = in.readOptionalLong(); timeout = TimeValue.timeValueMillis(in.readVLong()); + if (in.getVersion().onOrAfter(Version.CURRENT)) { + jobId = in.readOptionalString(); + datafeedIndices = in.readList(StreamInput::readString); + } } DatafeedParams() { @@ -197,6 +208,9 @@ public DatafeedParams(StreamInput in) throws IOException { private long startTime; private Long endTime; private TimeValue timeout = TimeValue.timeValueSeconds(20); + private List datafeedIndices = Collections.emptyList(); + private String jobId; + public String getDatafeedId() { return datafeedId; @@ -226,6 +240,22 @@ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public List getDatafeedIndices() { + return datafeedIndices; + } + + public void setDatafeedIndices(List datafeedIndices) { + this.datafeedIndices = datafeedIndices; + } + @Override public String getWriteableName() { return MlTasks.DATAFEED_TASK_NAME; @@ -242,6 +272,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(startTime); out.writeOptionalLong(endTime); out.writeVLong(timeout.millis()); + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalString(jobId); + out.writeStringList(datafeedIndices); + } } @Override @@ -253,13 +287,19 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(END_TIME.getPreferredName(), String.valueOf(endTime)); } builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep()); + if (jobId != null) { + builder.field(Job.ID.getPreferredName(), jobId); + } + if (datafeedIndices.isEmpty() == false) { + builder.field(INDICES.getPreferredName(), datafeedIndices); + } builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(datafeedId, startTime, endTime, timeout); + return Objects.hash(datafeedId, startTime, endTime, timeout, jobId, datafeedIndices); } @Override @@ -274,7 +314,9 @@ public boolean equals(Object obj) { return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(startTime, other.startTime) && Objects.equals(endTime, other.endTime) && - Objects.equals(timeout, other.timeout); + Objects.equals(timeout, other.timeout) && + Objects.equals(jobId, other.jobId) && + Objects.equals(datafeedIndices, other.datafeedIndices); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java index 24a6dbacfada5..79bfcde76e067 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; +import java.util.Arrays; public class DatafeedParamsTests extends AbstractSerializingTestCase { @Override @@ -28,6 +29,13 @@ public static StartDatafeedAction.DatafeedParams createDatafeedParams() { if (randomBoolean()) { params.setTimeout(TimeValue.timeValueMillis(randomNonNegativeLong())); } + if (randomBoolean()) { + params.setJobId(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + params.setDatafeedIndices(Arrays.asList(randomAlphaOfLength(10), randomAlphaOfLength(10))); + } + return params; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index a13b705f12408..2da8700971adb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -408,7 +408,8 @@ public Collection createComponents(Client client, ClusterService cluster jobManager, jobResultsProvider, jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory, xContentRegistry, auditor); this.autodetectProcessManager.set(autodetectProcessManager); - DatafeedJobBuilder datafeedJobBuilder = new DatafeedJobBuilder(client, jobResultsProvider, auditor, System::currentTimeMillis); + DatafeedJobBuilder datafeedJobBuilder = new DatafeedJobBuilder(client, settings, xContentRegistry, + auditor, System::currentTimeMillis); DatafeedManager datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder, System::currentTimeMillis, auditor); this.datafeedManager.set(datafeedManager); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index 5df11f02a3610..c7e867bb7de70 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -12,15 +12,13 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.xpack.ml.notifications.Auditor; import java.util.Objects; @@ -89,16 +87,20 @@ public void clusterChanged(ClusterChangedEvent event) { auditor.info(jobId, "Opening job on node [" + node.toString() + "]"); } } else if (MlTasks.DATAFEED_TASK_NAME.equals(currentTask.getTaskName())) { - String datafeedId = ((StartDatafeedAction.DatafeedParams) currentTask.getParams()).getDatafeedId(); - DatafeedConfig datafeedConfig = MlMetadata.getMlMetadata(event.state()).getDatafeed(datafeedId); + StartDatafeedAction.DatafeedParams datafeedParams = (StartDatafeedAction.DatafeedParams) currentTask.getParams(); + String jobId = datafeedParams.getJobId(); if (currentAssignment.getExecutorNode() == null) { - String msg = "No node found to start datafeed [" + datafeedId +"]. Reasons [" + + String msg = "No node found to start datafeed [" + datafeedParams.getDatafeedId() +"]. Reasons [" + currentAssignment.getExplanation() + "]"; - logger.warn("[{}] {}", datafeedConfig.getJobId(), msg); - auditor.warning(datafeedConfig.getJobId(), msg); + logger.warn("[{}] {}", jobId, msg); + if (jobId != null) { + auditor.warning(jobId, msg); + } } else { DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); - auditor.info(datafeedConfig.getJobId(), "Starting datafeed [" + datafeedId + "] on node [" + node + "]"); + if (jobId != null) { + auditor.info(jobId, "Starting datafeed [" + datafeedParams.getDatafeedId() + "] on node [" + node + "]"); + } } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index 578f4ee5f983b..aa416b6fa1d54 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -35,7 +35,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -48,10 +47,14 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.DatafeedNodeSelector; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Predicate; /* This class extends from TransportMasterNodeAction for cluster state observing purposes. @@ -67,34 +70,30 @@ public class TransportStartDatafeedAction extends TransportMasterNodeAction listener) { StartDatafeedAction.DatafeedParams params = request.getParams(); - if (licenseState.isMachineLearningAllowed()) { - - ActionListener> waitForTaskListener = - new ActionListener>() { - @Override - public void onResponse(PersistentTasksCustomMetaData.PersistentTask - persistentTask) { - waitForDatafeedStarted(persistentTask.getId(), params, listener); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceAlreadyExistsException) { - logger.debug("datafeed already started", e); - e = new ElasticsearchStatusException("cannot start datafeed [" + params.getDatafeedId() + - "] because it has already been started", RestStatus.CONFLICT); - } - listener.onFailure(e); - } - }; - - // Verify data extractor factory can be created, then start persistent task - MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - validate(params.getDatafeedId(), mlMetadata, tasks); - DatafeedConfig datafeed = mlMetadata.getDatafeed(params.getDatafeedId()); - Job job = mlMetadata.getJobs().get(datafeed.getJobId()); - - if (RemoteClusterLicenseChecker.containsRemoteIndex(datafeed.getIndices())) { - final RemoteClusterLicenseChecker remoteClusterLicenseChecker = - new RemoteClusterLicenseChecker(client, XPackLicenseState::isMachineLearningAllowedForOperationMode); - remoteClusterLicenseChecker.checkRemoteClusterLicenses( - RemoteClusterLicenseChecker.remoteClusterAliases(datafeed.getIndices()), - ActionListener.wrap( - response -> { - if (response.isSuccess() == false) { - listener.onFailure(createUnlicensedError(datafeed.getId(), response)); - } else { - createDataExtractor(job, datafeed, params, waitForTaskListener); - } - }, - e -> listener.onFailure( - createUnknownLicenseError( - datafeed.getId(), RemoteClusterLicenseChecker.remoteIndices(datafeed.getIndices()), e)) - )); - } else { - createDataExtractor(job, datafeed, params, waitForTaskListener); - } - } else { + if (licenseState.isMachineLearningAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); + return; } + + AtomicReference datafeedConfigHolder = new AtomicReference<>(); + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + + ActionListener> waitForTaskListener = + new ActionListener>() { + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask + persistentTask) { + waitForDatafeedStarted(persistentTask.getId(), params, listener); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + logger.debug("datafeed already started", e); + e = new ElasticsearchStatusException("cannot start datafeed [" + params.getDatafeedId() + + "] because it has already been started", RestStatus.CONFLICT); + } + listener.onFailure(e); + } + }; + + // Verify data extractor factory can be created, then start persistent task + Consumer createDataExtrator = job -> { + if (RemoteClusterLicenseChecker.containsRemoteIndex(params.getDatafeedIndices())) { + final RemoteClusterLicenseChecker remoteClusterLicenseChecker = + new RemoteClusterLicenseChecker(client, XPackLicenseState::isMachineLearningAllowedForOperationMode); + remoteClusterLicenseChecker.checkRemoteClusterLicenses( + RemoteClusterLicenseChecker.remoteClusterAliases(params.getDatafeedIndices()), + ActionListener.wrap( + response -> { + if (response.isSuccess() == false) { + listener.onFailure(createUnlicensedError(params.getDatafeedId(), response)); + } else { + createDataExtractor(job, datafeedConfigHolder.get(), params, waitForTaskListener); + } + }, + e -> listener.onFailure( + createUnknownLicenseError( + params.getDatafeedId(), + RemoteClusterLicenseChecker.remoteIndices(params.getDatafeedIndices()), e)) + ) + ); + } else { + createDataExtractor(job, datafeedConfigHolder.get(), params, waitForTaskListener); + } + }; + + ActionListener jobListener = ActionListener.wrap( + jobBuilder -> { + try { + Job job = jobBuilder.build(); + validate(job, datafeedConfigHolder.get(), tasks); + createDataExtrator.accept(job); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + ); + + ActionListener datafeedListener = ActionListener.wrap( + datafeedBuilder -> { + try { + DatafeedConfig datafeedConfig = datafeedBuilder.build(); + params.setDatafeedIndices(datafeedConfig.getIndices()); + params.setJobId(datafeedConfig.getJobId()); + datafeedConfigHolder.set(datafeedConfig); + jobConfigProvider.getJob(datafeedConfig.getJobId(), jobListener); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + ); + + datafeedConfigProvider.getDatafeedConfig(params.getDatafeedId(), datafeedListener); } private void createDataExtractor(Job job, DatafeedConfig datafeed, StartDatafeedAction.DatafeedParams params, @@ -280,14 +310,14 @@ public StartDatafeedPersistentTasksExecutor(Settings settings, DatafeedManager d @Override public PersistentTasksCustomMetaData.Assignment getAssignment(StartDatafeedAction.DatafeedParams params, ClusterState clusterState) { - return new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId()).selectNode(); + return new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId(), params.getJobId(), + params.getDatafeedIndices()).selectNode(); } @Override public void validate(StartDatafeedAction.DatafeedParams params, ClusterState clusterState) { - PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - TransportStartDatafeedAction.validate(params.getDatafeedId(), MlMetadata.getMlMetadata(clusterState), tasks); - new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId(), params.getJobId(), params.getDatafeedIndices()) + .checkDatafeedTaskCanBeCreated(); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java index 1fa402f4e2485..20900c3f0d8e7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java @@ -82,6 +82,10 @@ boolean isIsolated() { return isIsolated; } + public String getJobId() { + return jobId; + } + Long runLookBack(long startTime, Long endTime) throws Exception { lookbackStartTimeMs = skipToStartTime(startTime); Optional endMs = Optional.ofNullable(endTime); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java index efe332346efec..2f3c93be92b3f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java @@ -8,93 +8,154 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import java.util.Collections; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; public class DatafeedJobBuilder { private final Client client; - private final JobResultsProvider jobResultsProvider; + private final Settings settings; + private final NamedXContentRegistry xContentRegistry; private final Auditor auditor; private final Supplier currentTimeSupplier; - public DatafeedJobBuilder(Client client, JobResultsProvider jobResultsProvider, Auditor auditor, Supplier currentTimeSupplier) { + public DatafeedJobBuilder(Client client, Settings settings, NamedXContentRegistry xContentRegistry, + Auditor auditor, Supplier currentTimeSupplier) { this.client = client; - this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider); + this.settings = Objects.requireNonNull(settings); + this.xContentRegistry = Objects.requireNonNull(xContentRegistry); this.auditor = Objects.requireNonNull(auditor); this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier); } - void build(Job job, DatafeedConfig datafeed, ActionListener listener) { + void build(String datafeedId, ActionListener listener) { + + JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings); + JobConfigProvider jobConfigProvider = new JobConfigProvider(client, settings); + DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry); + + build(datafeedId, jobResultsProvider, jobConfigProvider, datafeedConfigProvider, listener); + } + + /** + * For testing only. + * Use {@link #build(String, ActionListener)} instead + */ + void build(String datafeedId, JobResultsProvider jobResultsProvider, JobConfigProvider jobConfigProvider, + DatafeedConfigProvider datafeedConfigProvider, ActionListener listener) { + + AtomicReference jobHolder = new AtomicReference<>(); + AtomicReference datafeedConfigHolder = new AtomicReference<>(); // Step 5. Build datafeed job object Consumer contextHanlder = context -> { - TimeValue frequency = getFrequencyOrDefault(datafeed, job); - TimeValue queryDelay = datafeed.getQueryDelay(); - DatafeedJob datafeedJob = new DatafeedJob(job.getId(), buildDataDescription(job), frequency.millis(), queryDelay.millis(), + TimeValue frequency = getFrequencyOrDefault(datafeedConfigHolder.get(), jobHolder.get()); + TimeValue queryDelay = datafeedConfigHolder.get().getQueryDelay(); + DatafeedJob datafeedJob = new DatafeedJob(jobHolder.get().getId(), buildDataDescription(jobHolder.get()), + frequency.millis(), queryDelay.millis(), context.dataExtractorFactory, client, auditor, currentTimeSupplier, context.latestFinalBucketEndMs, context.latestRecordTimeMs); + listener.onResponse(datafeedJob); }; final Context context = new Context(); - // Step 4. Context building complete - invoke final listener + // Context building complete - invoke final listener ActionListener dataExtractorFactoryHandler = ActionListener.wrap( dataExtractorFactory -> { context.dataExtractorFactory = dataExtractorFactory; contextHanlder.accept(context); }, e -> { - auditor.error(job.getId(), e.getMessage()); + auditor.error(jobHolder.get().getId(), e.getMessage()); listener.onFailure(e); } ); - // Step 3. Create data extractor factory + // Create data extractor factory Consumer dataCountsHandler = dataCounts -> { if (dataCounts.getLatestRecordTimeStamp() != null) { context.latestRecordTimeMs = dataCounts.getLatestRecordTimeStamp().getTime(); } - DataExtractorFactory.create(client, datafeed, job, dataExtractorFactoryHandler); + DataExtractorFactory.create(client, datafeedConfigHolder.get(), jobHolder.get(), dataExtractorFactoryHandler); }; - // Step 2. Collect data counts + // Collect data counts Consumer> bucketsHandler = buckets -> { if (buckets.results().size() == 1) { - TimeValue bucketSpan = job.getAnalysisConfig().getBucketSpan(); + TimeValue bucketSpan = jobHolder.get().getAnalysisConfig().getBucketSpan(); context.latestFinalBucketEndMs = buckets.results().get(0).getTimestamp().getTime() + bucketSpan.millis() - 1; } - jobResultsProvider.dataCounts(job.getId(), dataCountsHandler, listener::onFailure); + jobResultsProvider.dataCounts(jobHolder.get().getId(), dataCountsHandler, listener::onFailure); }; - // Step 1. Collect latest bucket - BucketsQueryBuilder latestBucketQuery = new BucketsQueryBuilder() - .sortField(Result.TIMESTAMP.getPreferredName()) - .sortDescending(true).size(1) - .includeInterim(false); - jobResultsProvider.bucketsViaInternalClient(job.getId(), latestBucketQuery, bucketsHandler, e -> { - if (e instanceof ResourceNotFoundException) { - QueryPage empty = new QueryPage<>(Collections.emptyList(), 0, Bucket.RESULT_TYPE_FIELD); - bucketsHandler.accept(empty); - } else { - listener.onFailure(e); - } - }); + // Collect latest bucket + Consumer jobIdConsumer = jobId -> { + BucketsQueryBuilder latestBucketQuery = new BucketsQueryBuilder() + .sortField(Result.TIMESTAMP.getPreferredName()) + .sortDescending(true).size(1) + .includeInterim(false); + jobResultsProvider.bucketsViaInternalClient(jobId, latestBucketQuery, bucketsHandler, e -> { + if (e instanceof ResourceNotFoundException) { + QueryPage empty = new QueryPage<>(Collections.emptyList(), 0, Bucket.RESULT_TYPE_FIELD); + bucketsHandler.accept(empty); + } else { + listener.onFailure(e); + } + }); + }; + + // Get the job config and re-validate + // Re-validation is required as the config has been re-read since + // the previous validation + ActionListener jobConfigListener = ActionListener.wrap( + jobBuilder -> { + try { + jobHolder.set(jobBuilder.build()); + DatafeedJobValidator.validate(datafeedConfigHolder.get(), jobHolder.get()); + jobIdConsumer.accept(jobHolder.get().getId()); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + ); + + // Get the datafeed config + ActionListener datafeedConfigListener = ActionListener.wrap( + configBuilder -> { + try { + datafeedConfigHolder.set(configBuilder.build()); + jobConfigProvider.getJob(datafeedConfigHolder.get().getJobId(), jobConfigListener); + } catch (Exception e) { + listener.onFailure(e); + } + }, + listener::onFailure + ); + + datafeedConfigProvider.getDatafeedConfig(datafeedId, datafeedConfigListener); } private static TimeValue getFrequencyOrDefault(DatafeedConfig datafeed, Job job) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java index 3d4d66eba92a3..c801118fe0fa7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java @@ -18,19 +18,16 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction; import org.elasticsearch.xpack.ml.notifications.Auditor; @@ -48,9 +45,9 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskListener; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; -import static org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskListener; public class DatafeedManager extends AbstractComponent { @@ -77,17 +74,14 @@ public DatafeedManager(ThreadPool threadPool, Client client, ClusterService clus clusterService.addListener(taskRunner); } - public void run(TransportStartDatafeedAction.DatafeedTask task, Consumer taskHandler) { - String datafeedId = task.getDatafeedId(); - ClusterState state = clusterService.state(); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); - Job job = mlMetadata.getJobs().get(datafeed.getJobId()); + public void run(TransportStartDatafeedAction.DatafeedTask task, Consumer finishHandler) { + String datafeedId = task.getDatafeedId(); ActionListener datafeedJobHandler = ActionListener.wrap( datafeedJob -> { - Holder holder = new Holder(task, datafeed, datafeedJob, new ProblemTracker(auditor, job.getId()), taskHandler); + Holder holder = new Holder(task, datafeedId, datafeedJob, + new ProblemTracker(auditor, datafeedJob.getJobId()), finishHandler); runningDatafeedsOnThisNode.put(task.getAllocationId(), holder); task.updatePersistentTaskState(DatafeedState.STARTED, new ActionListener>() { @Override @@ -97,13 +91,13 @@ public void onResponse(PersistentTask persistentTask) { @Override public void onFailure(Exception e) { - taskHandler.accept(e); + finishHandler.accept(e); } }); - }, taskHandler::accept + }, finishHandler::accept ); - datafeedJobBuilder.build(job, datafeed, datafeedJobHandler); + datafeedJobBuilder.build(datafeedId, datafeedJobHandler); } public void stopDatafeed(TransportStartDatafeedAction.DatafeedTask task, String reason, TimeValue timeout) { @@ -158,7 +152,7 @@ private void innerRun(Holder holder, long startTime, Long endTime) { @Override public void onFailure(Exception e) { - logger.error("Failed lookback import for job [" + holder.datafeed.getJobId() + "]", e); + logger.error("Failed lookback import for job [" + holder.datafeedJob.getJobId() + "]", e); holder.stop("general_lookback_failure", TimeValue.timeValueSeconds(20), e); } @@ -188,17 +182,17 @@ protected void doRun() { } else { // Notify that a lookback-only run found no data String lookbackNoDataMsg = Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_LOOKBACK_NO_DATA); - logger.warn("[{}] {}", holder.datafeed.getJobId(), lookbackNoDataMsg); - auditor.warning(holder.datafeed.getJobId(), lookbackNoDataMsg); + logger.warn("[{}] {}", holder.datafeedJob.getJobId(), lookbackNoDataMsg); + auditor.warning(holder.datafeedJob.getJobId(), lookbackNoDataMsg); } } catch (Exception e) { - logger.error("Failed lookback import for job [" + holder.datafeed.getJobId() + "]", e); + logger.error("Failed lookback import for job [" + holder.datafeedJob.getJobId() + "]", e); holder.stop("general_lookback_failure", TimeValue.timeValueSeconds(20), e); return; } if (isolated == false) { if (next != null) { - doDatafeedRealtime(next, holder.datafeed.getJobId(), holder); + doDatafeedRealtime(next, holder.datafeedJob.getJobId(), holder); } else { holder.stop("no_realtime", TimeValue.timeValueSeconds(20), null); holder.problemTracker.finishReport(); @@ -276,29 +270,29 @@ public class Holder { private final TransportStartDatafeedAction.DatafeedTask task; private final long allocationId; - private final DatafeedConfig datafeed; + private final String datafeedId; // To ensure that we wait until loopback / realtime search has completed before we stop the datafeed private final ReentrantLock datafeedJobLock = new ReentrantLock(true); private final DatafeedJob datafeedJob; private final boolean autoCloseJob; private final ProblemTracker problemTracker; - private final Consumer handler; + private final Consumer finishHandler; volatile Future future; private volatile boolean isRelocating; - Holder(TransportStartDatafeedAction.DatafeedTask task, DatafeedConfig datafeed, DatafeedJob datafeedJob, - ProblemTracker problemTracker, Consumer handler) { + Holder(TransportStartDatafeedAction.DatafeedTask task, String datafeedId, DatafeedJob datafeedJob, + ProblemTracker problemTracker, Consumer finishHandler) { this.task = task; this.allocationId = task.getAllocationId(); - this.datafeed = datafeed; + this.datafeedId = datafeedId; this.datafeedJob = datafeedJob; this.autoCloseJob = task.isLookbackOnly(); this.problemTracker = problemTracker; - this.handler = handler; + this.finishHandler = finishHandler; } String getJobId() { - return datafeed.getJobId(); + return datafeedJob.getJobId(); } boolean isRunning() { @@ -314,23 +308,23 @@ public void stop(String source, TimeValue timeout, Exception e) { return; } - logger.info("[{}] attempt to stop datafeed [{}] for job [{}]", source, datafeed.getId(), datafeed.getJobId()); + logger.info("[{}] attempt to stop datafeed [{}] for job [{}]", source, datafeedId, datafeedJob.getJobId()); if (datafeedJob.stop()) { boolean acquired = false; try { - logger.info("[{}] try lock [{}] to stop datafeed [{}] for job [{}]...", source, timeout, datafeed.getId(), - datafeed.getJobId()); + logger.info("[{}] try lock [{}] to stop datafeed [{}] for job [{}]...", source, timeout, datafeedId, + datafeedJob.getJobId()); acquired = datafeedJobLock.tryLock(timeout.millis(), TimeUnit.MILLISECONDS); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); } finally { - logger.info("[{}] stopping datafeed [{}] for job [{}], acquired [{}]...", source, datafeed.getId(), - datafeed.getJobId(), acquired); + logger.info("[{}] stopping datafeed [{}] for job [{}], acquired [{}]...", source, datafeedId, + datafeedJob.getJobId(), acquired); runningDatafeedsOnThisNode.remove(allocationId); FutureUtils.cancel(future); - auditor.info(datafeed.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STOPPED)); - handler.accept(e); - logger.info("[{}] datafeed [{}] for job [{}] has been stopped{}", source, datafeed.getId(), datafeed.getJobId(), + auditor.info(datafeedJob.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STOPPED)); + finishHandler.accept(e); + logger.info("[{}] datafeed [{}] for job [{}] has been stopped{}", source, datafeedId, datafeedJob.getJobId(), acquired ? "" : ", but there may be pending tasks as the timeout [" + timeout.getStringRep() + "] expired"); if (autoCloseJob) { closeJob(); @@ -340,7 +334,7 @@ public void stop(String source, TimeValue timeout, Exception e) { } } } else { - logger.info("[{}] datafeed [{}] for job [{}] was already stopped", source, datafeed.getId(), datafeed.getJobId()); + logger.info("[{}] datafeed [{}] for job [{}] was already stopped", source, datafeedId, datafeedJob.getJobId()); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java index ce3f611b2227a..a45bfc7822071 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java @@ -14,9 +14,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.license.RemoteClusterLicenseChecker; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -28,16 +26,20 @@ public class DatafeedNodeSelector { private static final Logger LOGGER = Loggers.getLogger(DatafeedNodeSelector.class); - private final DatafeedConfig datafeed; + private final String datafeedId; + private final String jobId; + private final List datafeedIndices; private final PersistentTasksCustomMetaData.PersistentTask jobTask; private final ClusterState clusterState; private final IndexNameExpressionResolver resolver; - public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolver resolver, String datafeedId) { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolver resolver, String datafeedId, + String jobId, List datafeedIndices) { PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - this.datafeed = mlMetadata.getDatafeed(datafeedId); - this.jobTask = MlTasks.getJobTask(datafeed.getJobId(), tasks); + this.datafeedId = datafeedId; + this.jobId = jobId; + this.datafeedIndices = datafeedIndices; + this.jobTask = MlTasks.getJobTask(jobId, tasks); this.clusterState = Objects.requireNonNull(clusterState); this.resolver = Objects.requireNonNull(resolver); } @@ -45,8 +47,8 @@ public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolv public void checkDatafeedTaskCanBeCreated() { AssignmentFailure assignmentFailure = checkAssignment(); if (assignmentFailure != null && assignmentFailure.isCriticalForTaskCreation) { - String msg = "No node found to start datafeed [" + datafeed.getId() + "], allocation explanation [" + assignmentFailure.reason - + "]"; + String msg = "No node found to start datafeed [" + datafeedId + "], " + + "allocation explanation [" + assignmentFailure.reason + "]"; LOGGER.debug(msg); throw ExceptionsHelper.conflictStatusException(msg); } @@ -64,7 +66,7 @@ public PersistentTasksCustomMetaData.Assignment selectNode() { @Nullable private AssignmentFailure checkAssignment() { PriorityFailureCollector priorityFailureCollector = new PriorityFailureCollector(); - priorityFailureCollector.add(verifyIndicesActive(datafeed)); + priorityFailureCollector.add(verifyIndicesActive()); JobTaskState jobTaskState = null; JobState jobState = JobState.CLOSED; @@ -75,13 +77,14 @@ private AssignmentFailure checkAssignment() { if (jobState.isAnyOf(JobState.OPENING, JobState.OPENED) == false) { // lets try again later when the job has been opened: - String reason = "cannot start datafeed [" + datafeed.getId() + "], because job's [" + datafeed.getJobId() + - "] state is [" + jobState + "] while state [" + JobState.OPENED + "] is required"; + String reason = "cannot start datafeed [" + datafeedId + "], because the job's [" + jobId + + "] state is [" + jobState + "] while state [" + JobState.OPENED + "] is required"; priorityFailureCollector.add(new AssignmentFailure(reason, true)); } if (jobTaskState != null && jobTaskState.isStatusStale(jobTask)) { - String reason = "cannot start datafeed [" + datafeed.getId() + "], job [" + datafeed.getJobId() + "] state is stale"; + String reason = "cannot start datafeed [" + datafeedId + "], because the job's [" + jobId + + "] state is stale"; priorityFailureCollector.add(new AssignmentFailure(reason, true)); } @@ -89,9 +92,8 @@ private AssignmentFailure checkAssignment() { } @Nullable - private AssignmentFailure verifyIndicesActive(DatafeedConfig datafeed) { - List indices = datafeed.getIndices(); - for (String index : indices) { + private AssignmentFailure verifyIndicesActive() { + for (String index : datafeedIndices) { if (RemoteClusterLicenseChecker.isRemoteIndex(index)) { // We cannot verify remote indices @@ -99,7 +101,7 @@ private AssignmentFailure verifyIndicesActive(DatafeedConfig datafeed) { } String[] concreteIndices; - String reason = "cannot start datafeed [" + datafeed.getId() + "] because index [" + String reason = "cannot start datafeed [" + datafeedId + "] because index [" + index + "] does not exist, is closed, or is still initializing."; try { @@ -115,7 +117,7 @@ private AssignmentFailure verifyIndicesActive(DatafeedConfig datafeed) { for (String concreteIndex : concreteIndices) { IndexRoutingTable routingTable = clusterState.getRoutingTable().index(concreteIndex); if (routingTable == null || !routingTable.allPrimaryShardsActive()) { - reason = "cannot start datafeed [" + datafeed.getId() + "] because index [" + reason = "cannot start datafeed [" + datafeedId + "] because index [" + concreteIndex + "] does not have all primary shards active yet."; return new AssignmentFailure(reason, false); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java index 3055dc2bb37f9..2defead100774 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java @@ -32,7 +32,7 @@ public class MlAssignmentNotifierTests extends ESTestCase { - public void testClusterChanged_info() throws Exception { + public void testClusterChanged_info() { Auditor auditor = mock(Auditor.class); ClusterService clusterService = mock(ClusterService.class); MlAssignmentNotifier notifier = new MlAssignmentNotifier(Settings.EMPTY, auditor, clusterService); @@ -60,7 +60,7 @@ public void testClusterChanged_info() throws Exception { verifyNoMoreInteractions(auditor); } - public void testClusterChanged_warning() throws Exception { + public void testClusterChanged_warning() { Auditor auditor = mock(Auditor.class); ClusterService clusterService = mock(ClusterService.class); MlAssignmentNotifier notifier = new MlAssignmentNotifier(Settings.EMPTY, auditor, clusterService); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java index 610a5c1b92fb6..7afb048f7aedb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java @@ -7,11 +7,9 @@ package org.elasticsearch.xpack.ml.action; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; @@ -28,61 +26,33 @@ public class TransportStartDatafeedActionTests extends ESTestCase { - public void testValidate_GivenDatafeedIsMissing() { - Job job = DatafeedManagerTests.createDatafeedJob().build(new Date()); - MlMetadata mlMetadata = new MlMetadata.Builder() - .putJob(job, false) - .build(); - Exception e = expectThrows(ResourceNotFoundException.class, - () -> TransportStartDatafeedAction.validate("some-datafeed", mlMetadata, null)); - assertThat(e.getMessage(), equalTo("No datafeed with id [some-datafeed] exists")); - } - public void testValidate_jobClosed() { Job job1 = DatafeedManagerTests.createDatafeedJob().build(new Date()); - MlMetadata mlMetadata1 = new MlMetadata.Builder() - .putJob(job1, false) - .build(); PersistentTasksCustomMetaData tasks = PersistentTasksCustomMetaData.builder().build(); DatafeedConfig datafeedConfig1 = DatafeedManagerTests.createDatafeedConfig("foo-datafeed", "job_id").build(); - MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) - .putDatafeed(datafeedConfig1, Collections.emptyMap()) - .build(); Exception e = expectThrows(ElasticsearchStatusException.class, - () -> TransportStartDatafeedAction.validate("foo-datafeed", mlMetadata2, tasks)); + () -> TransportStartDatafeedAction.validate(job1, datafeedConfig1, tasks)); assertThat(e.getMessage(), equalTo("cannot start datafeed [foo-datafeed] because job [job_id] is closed")); } public void testValidate_jobOpening() { Job job1 = DatafeedManagerTests.createDatafeedJob().build(new Date()); - MlMetadata mlMetadata1 = new MlMetadata.Builder() - .putJob(job1, false) - .build(); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", INITIAL_ASSIGNMENT.getExecutorNode(), null, tasksBuilder); PersistentTasksCustomMetaData tasks = tasksBuilder.build(); DatafeedConfig datafeedConfig1 = DatafeedManagerTests.createDatafeedConfig("foo-datafeed", "job_id").build(); - MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) - .putDatafeed(datafeedConfig1, Collections.emptyMap()) - .build(); - TransportStartDatafeedAction.validate("foo-datafeed", mlMetadata2, tasks); + TransportStartDatafeedAction.validate(job1, datafeedConfig1, tasks); } public void testValidate_jobOpened() { Job job1 = DatafeedManagerTests.createDatafeedJob().build(new Date()); - MlMetadata mlMetadata1 = new MlMetadata.Builder() - .putJob(job1, false) - .build(); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", INITIAL_ASSIGNMENT.getExecutorNode(), JobState.OPENED, tasksBuilder); PersistentTasksCustomMetaData tasks = tasksBuilder.build(); DatafeedConfig datafeedConfig1 = DatafeedManagerTests.createDatafeedConfig("foo-datafeed", "job_id").build(); - MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) - .putDatafeed(datafeedConfig1, Collections.emptyMap()) - .build(); - TransportStartDatafeedAction.validate("foo-datafeed", mlMetadata2, tasks); + TransportStartDatafeedAction.validate(job1, datafeedConfig1, tasks); } public static TransportStartDatafeedAction.DatafeedTask createDatafeedTask(long id, String type, String action, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java index f37deef12d083..7595346d0c70d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java @@ -19,6 +19,8 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.Before; @@ -32,6 +34,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -41,8 +44,10 @@ public class DatafeedJobBuilderTests extends ESTestCase { private Client client; private Auditor auditor; - private JobResultsProvider jobResultsProvider; private Consumer taskHandler; + private JobResultsProvider jobResultsProvider; + private JobConfigProvider jobConfigProvider; + private DatafeedConfigProvider datafeedConfigProvider; private DatafeedJobBuilder datafeedJobBuilder; @@ -54,10 +59,10 @@ public void init() { when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); when(client.settings()).thenReturn(Settings.EMPTY); auditor = mock(Auditor.class); - jobResultsProvider = mock(JobResultsProvider.class); taskHandler = mock(Consumer.class); - datafeedJobBuilder = new DatafeedJobBuilder(client, jobResultsProvider, auditor, System::currentTimeMillis); + datafeedJobBuilder = new DatafeedJobBuilder(client, Settings.EMPTY, xContentRegistry(), auditor, System::currentTimeMillis); + jobResultsProvider = mock(JobResultsProvider.class); Mockito.doAnswer(invocationOnMock -> { String jobId = (String) invocationOnMock.getArguments()[0]; @SuppressWarnings("unchecked") @@ -72,6 +77,9 @@ public void init() { consumer.accept(new ResourceNotFoundException("dummy")); return null; }).when(jobResultsProvider).bucketsViaInternalClient(any(), any(), any(), any()); + + jobConfigProvider = mock(JobConfigProvider.class); + datafeedConfigProvider = mock(DatafeedConfigProvider.class); } public void testBuild_GivenScrollDatafeedAndNewJob() throws Exception { @@ -79,7 +87,8 @@ public void testBuild_GivenScrollDatafeedAndNewJob() throws Exception { dataDescription.setTimeField("time"); Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob(); jobBuilder.setDataDescription(dataDescription); - DatafeedConfig datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo").build(); + jobBuilder.setCreateTime(new Date()); + DatafeedConfig.Builder datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", jobBuilder.getId()); AtomicBoolean wasHandlerCalled = new AtomicBoolean(false); ActionListener datafeedJobHandler = ActionListener.wrap( @@ -91,7 +100,10 @@ public void testBuild_GivenScrollDatafeedAndNewJob() throws Exception { }, e -> fail() ); - datafeedJobBuilder.build(jobBuilder.build(new Date()), datafeed, datafeedJobHandler); + givenJob(jobBuilder); + givenDatafeed(datafeed); + + datafeedJobBuilder.build("datafeed1", jobResultsProvider, jobConfigProvider, datafeedConfigProvider, datafeedJobHandler); assertBusy(() -> wasHandlerCalled.get()); } @@ -101,7 +113,8 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestRecordTimestampAfter dataDescription.setTimeField("time"); Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob(); jobBuilder.setDataDescription(dataDescription); - DatafeedConfig datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo").build(); + jobBuilder.setCreateTime(new Date()); + DatafeedConfig.Builder datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", jobBuilder.getId()); givenLatestTimes(7_200_000L, 3_600_000L); @@ -115,7 +128,10 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestRecordTimestampAfter }, e -> fail() ); - datafeedJobBuilder.build(jobBuilder.build(new Date()), datafeed, datafeedJobHandler); + givenJob(jobBuilder); + givenDatafeed(datafeed); + + datafeedJobBuilder.build("datafeed1", jobResultsProvider, jobConfigProvider, datafeedConfigProvider, datafeedJobHandler); assertBusy(() -> wasHandlerCalled.get()); } @@ -125,7 +141,8 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestBucketAfterLatestRec dataDescription.setTimeField("time"); Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob(); jobBuilder.setDataDescription(dataDescription); - DatafeedConfig datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo").build(); + jobBuilder.setCreateTime(new Date()); + DatafeedConfig.Builder datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", jobBuilder.getId()); givenLatestTimes(3_800_000L, 3_600_000L); @@ -139,7 +156,10 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestBucketAfterLatestRec }, e -> fail() ); - datafeedJobBuilder.build(jobBuilder.build(new Date()), datafeed, datafeedJobHandler); + givenJob(jobBuilder); + givenDatafeed(datafeed); + + datafeedJobBuilder.build("datafeed1", jobResultsProvider, jobConfigProvider, datafeedConfigProvider, datafeedJobHandler); assertBusy(() -> wasHandlerCalled.get()); } @@ -149,7 +169,8 @@ public void testBuild_GivenBucketsRequestFails() { dataDescription.setTimeField("time"); Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob(); jobBuilder.setDataDescription(dataDescription); - DatafeedConfig datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo").build(); + jobBuilder.setCreateTime(new Date()); + DatafeedConfig.Builder datafeed = DatafeedManagerTests.createDatafeedConfig("datafeed1", jobBuilder.getId()); Exception error = new RuntimeException("error"); doAnswer(invocationOnMock -> { @@ -159,11 +180,34 @@ public void testBuild_GivenBucketsRequestFails() { return null; }).when(jobResultsProvider).bucketsViaInternalClient(any(), any(), any(), any()); - datafeedJobBuilder.build(jobBuilder.build(new Date()), datafeed, ActionListener.wrap(datafeedJob -> fail(), taskHandler)); + + givenJob(jobBuilder); + givenDatafeed(datafeed); + + datafeedJobBuilder.build("datafeed1", jobResultsProvider, jobConfigProvider, datafeedConfigProvider, + ActionListener.wrap(datafeedJob -> fail(), taskHandler)); verify(taskHandler).accept(error); } + private void givenJob(Job.Builder job) { + Mockito.doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener handler = (ActionListener) invocationOnMock.getArguments()[1]; + handler.onResponse(job); + return null; + }).when(jobConfigProvider).getJob(eq(job.getId()), any()); + } + + private void givenDatafeed(DatafeedConfig.Builder datafeed) { + Mockito.doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener handler = (ActionListener) invocationOnMock.getArguments()[1]; + handler.onResponse(datafeed); + return null; + }).when(datafeedConfigProvider).getDatafeedConfig(eq(datafeed.getId()), any()); + } + private void givenLatestTimes(long latestRecordTimestamp, long latestBucketTimestamp) { Mockito.doAnswer(invocationOnMock -> { String jobId = (String) invocationOnMock.getArguments()[0]; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java index a9dec7c66d4b6..54aa3ade8e1b9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java @@ -21,9 +21,10 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -34,11 +35,9 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.notifications.AuditMessage; import org.elasticsearch.xpack.core.ml.notifications.AuditorField; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.action.TransportStartDatafeedActionTests; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction.DatafeedTask; +import org.elasticsearch.xpack.ml.action.TransportStartDatafeedActionTests; import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.Before; @@ -79,11 +78,8 @@ public class DatafeedManagerTests extends ESTestCase { @Before @SuppressWarnings("unchecked") public void setUpTests() { - MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); - Job job = createDatafeedJob().build(new Date()); - mlMetadata.putJob(job, false); - DatafeedConfig datafeed = createDatafeedConfig("datafeed_id", job.getId()).build(); - mlMetadata.putDatafeed(datafeed, Collections.emptyMap()); + Job.Builder job = createDatafeedJob().setCreateTime(new Date()); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); PersistentTasksCustomMetaData tasks = tasksBuilder.build(); @@ -92,8 +88,7 @@ public void setUpTests() { Collections.emptyMap(), Collections.emptySet(), Version.CURRENT)) .build(); ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasks)) .nodes(nodes); clusterService = mock(ClusterService.class); @@ -124,13 +119,14 @@ public void setUpTests() { datafeedJob = mock(DatafeedJob.class); when(datafeedJob.isRunning()).thenReturn(true); when(datafeedJob.stop()).thenReturn(true); + when(datafeedJob.getJobId()).thenReturn(job.getId()); DatafeedJobBuilder datafeedJobBuilder = mock(DatafeedJobBuilder.class); doAnswer(invocationOnMock -> { @SuppressWarnings("rawtypes") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; listener.onResponse(datafeedJob); return null; - }).when(datafeedJobBuilder).build(any(), any(), any()); + }).when(datafeedJobBuilder).build(any(), any()); datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder, () -> currentTime, auditor); @@ -253,8 +249,7 @@ public void testDatafeedTaskWaitsUntilJobIsOpened() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder); ClusterState.Builder cs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); when(clusterService.state()).thenReturn(cs.build()); Consumer handler = mockConsumer(); @@ -268,8 +263,7 @@ public void testDatafeedTaskWaitsUntilJobIsOpened() { addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder); addJobTask("another_job", "node_id", JobState.OPENED, tasksBuilder); ClusterState.Builder anotherJobCs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", anotherJobCs.build(), cs.build())); @@ -279,8 +273,7 @@ public void testDatafeedTaskWaitsUntilJobIsOpened() { tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.OPENED, tasksBuilder); ClusterState.Builder jobOpenedCs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); capturedClusterStateListener.getValue().clusterChanged( new ClusterChangedEvent("_source", jobOpenedCs.build(), anotherJobCs.build())); @@ -293,8 +286,7 @@ public void testDatafeedTaskStopsBecauseJobFailedWhileOpening() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder); ClusterState.Builder cs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); when(clusterService.state()).thenReturn(cs.build()); Consumer handler = mockConsumer(); @@ -307,8 +299,7 @@ public void testDatafeedTaskStopsBecauseJobFailedWhileOpening() { tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.FAILED, tasksBuilder); ClusterState.Builder updatedCs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", updatedCs.build(), cs.build())); @@ -321,8 +312,7 @@ public void testDatafeedGetsStoppedWhileWaitingForJobToOpen() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder); ClusterState.Builder cs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); when(clusterService.state()).thenReturn(cs.build()); Consumer handler = mockConsumer(); @@ -339,8 +329,7 @@ public void testDatafeedGetsStoppedWhileWaitingForJobToOpen() { tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", "node_id", JobState.OPENED, tasksBuilder); ClusterState.Builder updatedCs = ClusterState.builder(clusterService.state()) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state())) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); + .metaData(new MetaData.Builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())); capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", cs.build(), updatedCs.build())); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 4b8ad1d08aed3..dfaf9f03c0dec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -26,13 +26,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.junit.Before; import java.net.InetAddress; @@ -52,7 +52,6 @@ public class DatafeedNodeSelectorTests extends ESTestCase { private IndexNameExpressionResolver resolver; private DiscoveryNodes nodes; private ClusterState clusterState; - private MlMetadata mlMetadata; private PersistentTasksCustomMetaData tasks; @Before @@ -65,11 +64,8 @@ public void init() { } public void testSelectNode_GivenJobIsOpened() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -77,17 +73,15 @@ public void testSelectNode_GivenJobIsOpened() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobIsOpening() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", null, tasksBuilder); @@ -95,41 +89,38 @@ public void testSelectNode_GivenJobIsOpening() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); } public void testNoJobTask() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); // Using wildcard index name to test for index resolving as well - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")); tasks = PersistentTasksCustomMetaData.builder().build(); givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); - assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id], because job's [job_id] state is " + + assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id], because the job's [job_id] state is " + "[closed] while state [opened] is required")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " - + "[cannot start datafeed [datafeed_id], because job's [job_id] state is [closed] while state [opened] is required]")); + + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [closed] while state [opened] is required]")); } public void testSelectNode_GivenJobFailedOrClosed() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); JobState jobState = randomFrom(JobState.FAILED, JobState.CLOSED); @@ -138,26 +129,25 @@ public void testSelectNode_GivenJobFailedOrClosed() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); - assertEquals("cannot start datafeed [datafeed_id], because job's [job_id] state is [" + jobState + + assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " - + "[cannot start datafeed [datafeed_id], because job's [job_id] state is [" + jobState + + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required]")); } public void testShardUnassigned() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); // Using wildcard index name to test for index resolving as well - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -168,22 +158,20 @@ public void testShardUnassigned() { givenClusterState("foo", 1, 0, states); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); } public void testShardNotAllActive() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); // Using wildcard index name to test for index resolving as well - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("fo*")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -195,21 +183,18 @@ public void testShardNotAllActive() { givenClusterState("foo", 2, 0, states); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); } public void testIndexDoesntExist() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo")), - Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -217,24 +202,22 @@ public void testIndexDoesntExist() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [not_foo] " + "does not exist, is closed, or is still initializing.")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); } public void testRemoteIndex() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("remote:foo")), - Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("remote:foo")); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -242,16 +225,14 @@ public void testRemoteIndex() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNotNull(result.getExecutorNode()); } public void testSelectNode_jobTaskStale() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")), Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("foo")); String nodeId = randomBoolean() ? "node_id2" : null; PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); @@ -262,44 +243,43 @@ public void testSelectNode_jobTaskStale() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + PersistentTasksCustomMetaData.Assignment result = + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertNull(result.getExecutorNode()); - assertEquals("cannot start datafeed [datafeed_id], job [job_id] state is stale", + assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is stale", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " - + "[cannot start datafeed [datafeed_id], job [job_id] state is stale]")); + + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is stale]")); tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id1", JobState.OPENED, tasksBuilder); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); - result = new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").selectNode(); + result = new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); assertEquals("node_id1", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { // Here we test that when there are 2 problems, the most critical gets reported first. // In this case job is Opening (non-critical) and the index does not exist (critical) - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); Job job = createScheduledJob("job_id").build(new Date()); - mlMetadataBuilder.putJob(job, false); - mlMetadataBuilder.putDatafeed(createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo")), - Collections.emptyMap()); - mlMetadata = mlMetadataBuilder.build(); + DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo")); - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), "node_id", JobState.OPENING, tasksBuilder); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, "datafeed_id").checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); } @@ -319,7 +299,6 @@ private void givenClusterState(String index, int numberOfShards, int numberOfRep clusterState = ClusterState.builder(new ClusterName("cluster_name")) .metaData(new MetaData.Builder() - .putCustom(MlMetadata.TYPE, mlMetadata) .putCustom(PersistentTasksCustomMetaData.TYPE, tasks) .put(indexMetaData, false)) .nodes(nodes) From 4e3d565dc9707abf358a1baf4081834aec8a1a36 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 19 Oct 2018 11:21:52 +0100 Subject: [PATCH 11/57] [ML] Job in Index: Stop and preview datafeed (#34605) --- .../TransportPreviewDatafeedAction.java | 75 +++++++++--------- .../action/TransportStopDatafeedAction.java | 78 ++++++++----------- .../TransportStopDatafeedActionTests.java | 77 +++--------------- 3 files changed, 84 insertions(+), 146 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java index cc3a34f20f570..d91a921f48aec 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -17,14 +16,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.io.BufferedReader; import java.io.InputStream; @@ -39,51 +37,56 @@ public class TransportPreviewDatafeedAction extends HandledTransportAction) PreviewDatafeedAction.Request::new); this.threadPool = threadPool; this.client = client; - this.clusterService = clusterService; + this.jobConfigProvider = jobConfigProvider; + this.datafeedConfigProvider = datafeedConfigProvider; } @Override protected void doExecute(Task task, PreviewDatafeedAction.Request request, ActionListener listener) { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterService.state()); - DatafeedConfig datafeed = mlMetadata.getDatafeed(request.getDatafeedId()); - if (datafeed == null) { - throw ExceptionsHelper.missingDatafeedException(request.getDatafeedId()); - } - Job job = mlMetadata.getJobs().get(datafeed.getJobId()); - if (job == null) { - throw ExceptionsHelper.missingJobException(datafeed.getJobId()); - } - DatafeedConfig.Builder previewDatafeed = buildPreviewDatafeed(datafeed); - Map headers = threadPool.getThreadContext().getHeaders().entrySet().stream() - .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - previewDatafeed.setHeaders(headers); - // NB: this is using the client from the transport layer, NOT the internal client. - // This is important because it means the datafeed search will fail if the user - // requesting the preview doesn't have permission to search the relevant indices. - DataExtractorFactory.create(client, previewDatafeed.build(), job, new ActionListener() { - @Override - public void onResponse(DataExtractorFactory dataExtractorFactory) { - DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); - threadPool.generic().execute(() -> previewDatafeed(dataExtractor, listener)); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + datafeedConfigProvider.getDatafeedConfig(request.getDatafeedId(), ActionListener.wrap( + datafeedConfigBuilder -> { + DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); + jobConfigProvider.getJob(datafeedConfig.getJobId(), ActionListener.wrap( + jobBuilder -> { + DatafeedConfig.Builder previewDatafeed = buildPreviewDatafeed(datafeedConfig); + Map headers = threadPool.getThreadContext().getHeaders().entrySet().stream() + .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + previewDatafeed.setHeaders(headers); + // NB: this is using the client from the transport layer, NOT the internal client. + // This is important because it means the datafeed search will fail if the user + // requesting the preview doesn't have permission to search the relevant indices. + DataExtractorFactory.create(client, previewDatafeed.build(), jobBuilder.build(), + new ActionListener() { + @Override + public void onResponse(DataExtractorFactory dataExtractorFactory) { + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); + threadPool.generic().execute(() -> previewDatafeed(dataExtractor, listener)); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + }, + listener::onFailure + )); + }, + listener::onFailure + )); } /** Visible for testing */ diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java index 1cc6d97158959..d46b085b527f0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java @@ -22,19 +22,17 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import java.io.IOException; import java.util.ArrayList; @@ -50,35 +48,35 @@ public class TransportStopDatafeedAction extends TransportTasksAction startedDatafeedIds, - List stoppingDatafeedIds) { + static void sortDatafeedIdsByTaskState(Set expandedDatafeedIds, + PersistentTasksCustomMetaData tasks, + List startedDatafeedIds, + List stoppingDatafeedIds) { - Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds()); for (String expandedDatafeedId : expandedDatafeedIds) { - validateDatafeedTask(expandedDatafeedId, mlMetadata); addDatafeedTaskIdAccordingToState(expandedDatafeedId, MlTasks.getDatafeedState(expandedDatafeedId, tasks), startedDatafeedIds, stoppingDatafeedIds); } @@ -102,20 +100,6 @@ private static void addDatafeedTaskIdAccordingToState(String datafeedId, } } - /** - * Validate the stop request. - * Throws an {@code ResourceNotFoundException} if there is no datafeed - * with id {@code datafeedId} - * @param datafeedId The datafeed Id - * @param mlMetadata ML meta data - */ - static void validateDatafeedTask(String datafeedId, MlMetadata mlMetadata) { - DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); - if (datafeed == null) { - throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); - } - } - @Override protected void doExecute(Task task, StopDatafeedAction.Request request, ActionListener listener) { final ClusterState state = clusterService.state(); @@ -130,23 +114,27 @@ protected void doExecute(Task task, StopDatafeedAction.Request request, ActionLi new ActionListenerResponseHandler<>(listener, StopDatafeedAction.Response::new)); } } else { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + datafeedConfigProvider.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds(), ActionListener.wrap( + expandedIds -> { + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - List startedDatafeeds = new ArrayList<>(); - List stoppingDatafeeds = new ArrayList<>(); - resolveDataFeedIds(request, mlMetadata, tasks, startedDatafeeds, stoppingDatafeeds); - if (startedDatafeeds.isEmpty() && stoppingDatafeeds.isEmpty()) { - listener.onResponse(new StopDatafeedAction.Response(true)); - return; - } - request.setResolvedStartedDatafeedIds(startedDatafeeds.toArray(new String[startedDatafeeds.size()])); + List startedDatafeeds = new ArrayList<>(); + List stoppingDatafeeds = new ArrayList<>(); + sortDatafeedIdsByTaskState(expandedIds, tasks, startedDatafeeds, stoppingDatafeeds); + if (startedDatafeeds.isEmpty() && stoppingDatafeeds.isEmpty()) { + listener.onResponse(new StopDatafeedAction.Response(true)); + return; + } + request.setResolvedStartedDatafeedIds(startedDatafeeds.toArray(new String[startedDatafeeds.size()])); - if (request.isForce()) { - forceStopDatafeed(request, listener, tasks, startedDatafeeds); - } else { - normalStopDatafeed(task, request, listener, tasks, startedDatafeeds, stoppingDatafeeds); - } + if (request.isForce()) { + forceStopDatafeed(request, listener, tasks, startedDatafeeds); + } else { + normalStopDatafeed(task, request, listener, tasks, startedDatafeeds, stoppingDatafeeds); + } + }, + listener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java index d8b1d28153688..da390b6106245 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java @@ -5,112 +5,59 @@ */ package org.elasticsearch.xpack.ml.action; -import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Date; +import java.util.HashSet; import java.util.List; -import static org.elasticsearch.xpack.ml.datafeed.DatafeedManagerTests.createDatafeedConfig; -import static org.elasticsearch.xpack.ml.datafeed.DatafeedManagerTests.createDatafeedJob; -import static org.hamcrest.Matchers.equalTo; - public class TransportStopDatafeedActionTests extends ESTestCase { - public void testValidate() { - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, - new StartDatafeedAction.DatafeedParams("foo", 0L), new PersistentTasksCustomMetaData.Assignment("node_id", "")); - tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED); - tasksBuilder.build(); - - Job job = createDatafeedJob().build(new Date()); - MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build(); - Exception e = expectThrows(ResourceNotFoundException.class, - () -> TransportStopDatafeedAction.validateDatafeedTask("foo", mlMetadata1)); - assertThat(e.getMessage(), equalTo("No datafeed with id [foo] exists")); - - DatafeedConfig datafeedConfig = createDatafeedConfig("foo", "job_id").build(); - MlMetadata mlMetadata2 = new MlMetadata.Builder().putJob(job, false) - .putDatafeed(datafeedConfig, Collections.emptyMap()) - .build(); - TransportStopDatafeedAction.validateDatafeedTask("foo", mlMetadata2); - } - - public void testResolveDataFeedIds_GivenDatafeedId() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); + public void testSortDatafeedIdsByTaskState_GivenDatafeedId() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addTask("datafeed_1", 0L, "node-1", DatafeedState.STARTED, tasksBuilder); - Job job = BaseMlIntegTestCase.createScheduledJob("job_id_1").build(new Date()); - DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed_1", "job_id_1").build(); - mlMetadataBuilder.putJob(job, false).putDatafeed(datafeedConfig, Collections.emptyMap()); - addTask("datafeed_2", 0L, "node-1", DatafeedState.STOPPED, tasksBuilder); - job = BaseMlIntegTestCase.createScheduledJob("job_id_2").build(new Date()); - datafeedConfig = createDatafeedConfig("datafeed_2", "job_id_2").build(); - mlMetadataBuilder.putJob(job, false).putDatafeed(datafeedConfig, Collections.emptyMap()); - PersistentTasksCustomMetaData tasks = tasksBuilder.build(); - MlMetadata mlMetadata = mlMetadataBuilder.build(); List startedDatafeeds = new ArrayList<>(); List stoppingDatafeeds = new ArrayList<>(); - TransportStopDatafeedAction.resolveDataFeedIds(new StopDatafeedAction.Request("datafeed_1"), mlMetadata, tasks, startedDatafeeds, - stoppingDatafeeds); + TransportStopDatafeedAction.sortDatafeedIdsByTaskState( + Collections.singleton("datafeed_1"), tasks, startedDatafeeds, stoppingDatafeeds); assertEquals(Collections.singletonList("datafeed_1"), startedDatafeeds); assertEquals(Collections.emptyList(), stoppingDatafeeds); startedDatafeeds.clear(); stoppingDatafeeds.clear(); - TransportStopDatafeedAction.resolveDataFeedIds(new StopDatafeedAction.Request("datafeed_2"), mlMetadata, tasks, startedDatafeeds, - stoppingDatafeeds); + TransportStopDatafeedAction.sortDatafeedIdsByTaskState( + Collections.singleton("datafeed_2"), tasks, startedDatafeeds, stoppingDatafeeds); assertEquals(Collections.emptyList(), startedDatafeeds); assertEquals(Collections.emptyList(), stoppingDatafeeds); } - public void testResolveDataFeedIds_GivenAll() { - MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(); + public void testSortDatafeedIdsByTaskState_GivenAll() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addTask("datafeed_1", 0L, "node-1", DatafeedState.STARTED, tasksBuilder); - Job job = BaseMlIntegTestCase.createScheduledJob("job_id_1").build(new Date()); - DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed_1", "job_id_1").build(); - mlMetadataBuilder.putJob(job, false).putDatafeed(datafeedConfig, Collections.emptyMap()); - addTask("datafeed_2", 0L, "node-1", DatafeedState.STOPPED, tasksBuilder); - job = BaseMlIntegTestCase.createScheduledJob("job_id_2").build(new Date()); - datafeedConfig = createDatafeedConfig("datafeed_2", "job_id_2").build(); - mlMetadataBuilder.putJob(job, false).putDatafeed(datafeedConfig, Collections.emptyMap()); - addTask("datafeed_3", 0L, "node-1", DatafeedState.STOPPING, tasksBuilder); - job = BaseMlIntegTestCase.createScheduledJob("job_id_3").build(new Date()); - datafeedConfig = createDatafeedConfig("datafeed_3", "job_id_3").build(); - mlMetadataBuilder.putJob(job, false).putDatafeed(datafeedConfig, Collections.emptyMap()); - PersistentTasksCustomMetaData tasks = tasksBuilder.build(); - MlMetadata mlMetadata = mlMetadataBuilder.build(); List startedDatafeeds = new ArrayList<>(); List stoppingDatafeeds = new ArrayList<>(); - TransportStopDatafeedAction.resolveDataFeedIds(new StopDatafeedAction.Request("_all"), mlMetadata, tasks, startedDatafeeds, - stoppingDatafeeds); + TransportStopDatafeedAction.sortDatafeedIdsByTaskState( + new HashSet<>(Arrays.asList("datafeed_1", "datafeed_2", "datafeed_3")), tasks, startedDatafeeds, stoppingDatafeeds); assertEquals(Collections.singletonList("datafeed_1"), startedDatafeeds); assertEquals(Collections.singletonList("datafeed_3"), stoppingDatafeeds); startedDatafeeds.clear(); stoppingDatafeeds.clear(); - TransportStopDatafeedAction.resolveDataFeedIds(new StopDatafeedAction.Request("datafeed_2"), mlMetadata, tasks, startedDatafeeds, + TransportStopDatafeedAction.sortDatafeedIdsByTaskState(Collections.singleton("datafeed_2"), tasks, startedDatafeeds, stoppingDatafeeds); assertEquals(Collections.emptyList(), startedDatafeeds); assertEquals(Collections.emptyList(), stoppingDatafeeds); From cef9f30a45ed067107f2408bdcbd58129e29b9e6 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 19 Oct 2018 13:04:51 +0100 Subject: [PATCH 12/57] [ML] Delete job document (#34595) --- .../persistence/AnomalyDetectorsIndex.java | 12 -- .../ml/action/TransportDeleteJobAction.java | 147 +++++++++--------- .../ml/job/persistence/JobConfigProvider.java | 56 ++++++- .../action/TransportDeleteJobActionTests.java | 37 ----- .../action/TransportOpenJobActionTests.java | 2 +- .../ml/integration/JobConfigProviderIT.java | 34 +++- 6 files changed, 153 insertions(+), 135 deletions(-) delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 6cf4aee2a9672..b7b104e35cdec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -5,9 +5,6 @@ */ package org.elasticsearch.xpack.core.ml.job.persistence; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.xpack.core.ml.MlMetadata; - /** * Methods for handling index naming related functions */ @@ -40,15 +37,6 @@ public static String resultsWriteAlias(String jobId) { return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + ".write-" + jobId; } - /** - * Retrieves the currently defined physical index from the job state - * @param jobId Job Id - * @return The index name - */ - public static String getPhysicalIndexFromState(ClusterState state, String jobId) { - return MlMetadata.getMlMetadata(state).getJobs().get(jobId).getResultsIndexName(); - } - /** * The name of the default index where a job's state is stored * @return The index name diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 89f42d622411f..d8e8b8f8178f7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -24,14 +24,11 @@ import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; import org.elasticsearch.client.ParentTaskAssigningClient; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Nullable; @@ -52,30 +49,36 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; import org.elasticsearch.xpack.core.ml.action.KillProcessAction; import org.elasticsearch.xpack.core.ml.action.util.PageParams; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobState; +import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.utils.MlIndicesUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -89,6 +92,8 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction(); } @@ -137,6 +145,10 @@ protected void masterOperation(Task task, DeleteJobAction.Request request, Clust ActionListener listener) { logger.debug("Deleting job '{}'", request.getJobId()); + if (request.isForce() == false) { + checkJobIsNotOpen(request.getJobId(), state); + } + TaskId taskId = new TaskId(clusterService.localNode().getId(), task.getId()); ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, taskId); @@ -175,7 +187,7 @@ protected void masterOperation(Task task, DeleteJobAction.Request request, Clust finalListener.onFailure(e); }); - markJobAsDeleting(request.getJobId(), markAsDeletingListener, request.isForce()); + markJobAsDeletingIfNotUsed(request.getJobId(), markAsDeletingListener); } private void notifyListeners(String jobId, @Nullable AcknowledgedResponse ack, @Nullable Exception error) { @@ -211,33 +223,15 @@ private void normalDeleteJob(ParentTaskAssigningClient parentTaskClient, DeleteJ } }; - // Step 3. When the physical storage has been deleted, remove from Cluster State + // Step 3. When the physical storage has been deleted, delete the job config document // ------- - CheckedConsumer deleteJobStateHandler = response -> clusterService.submitStateUpdateTask( - "delete-job-" + jobId, - new AckedClusterStateUpdateTask(request, ActionListener.wrap(apiResponseHandler, listener::onFailure)) { - - @Override - protected Boolean newResponse(boolean acknowledged) { - return acknowledged && response; - } - - @Override - public ClusterState execute(ClusterState currentState) { - MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(currentState); - if (currentMlMetadata.getJobs().containsKey(jobId) == false) { - // We wouldn't have got here if the job never existed so - // the Job must have been deleted by another action. - // Don't error in this case - return currentState; - } - - MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata); - builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); - return buildNewClusterState(currentState, builder); - } - }); - + // Don't report an error if the document has already been deleted + CheckedConsumer deleteJobStateHandler = response -> jobConfigProvider.deleteJob(jobId, false, + ActionListener.wrap( + deleteResponse -> apiResponseHandler.accept(Boolean.TRUE), + listener::onFailure + ) + ); // Step 2. Remove the job from any calendars CheckedConsumer removeFromCalendarsHandler = response -> jobResultsProvider.removeJobFromCalendars(jobId, @@ -251,26 +245,26 @@ public ClusterState execute(ClusterState currentState) { private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, String jobId, CheckedConsumer finishedHandler, Consumer failureHandler) { - final String indexName = AnomalyDetectorsIndex.getPhysicalIndexFromState(clusterService.state(), jobId); - final String indexPattern = indexName + "-*"; + AtomicReference indexName = new AtomicReference<>(); final ActionListener completionHandler = ActionListener.wrap( response -> finishedHandler.accept(response.isAcknowledged()), failureHandler); - // Step 7. If we did not drop the index and after DBQ state done, we delete the aliases + // Step 8. If we did not drop the index and after DBQ state done, we delete the aliases ActionListener dbqHandler = ActionListener.wrap( bulkByScrollResponse -> { if (bulkByScrollResponse == null) { // no action was taken by DBQ, assume Index was deleted completionHandler.onResponse(new AcknowledgedResponse(true)); } else { if (bulkByScrollResponse.isTimedOut()) { - logger.warn("[{}] DeleteByQuery for indices [{}, {}] timed out.", jobId, indexName, indexPattern); + logger.warn("[{}] DeleteByQuery for indices [{}, {}] timed out.", jobId, indexName.get(), + indexName.get() + "-*"); } if (!bulkByScrollResponse.getBulkFailures().isEmpty()) { logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery on indices [{}, {}].", jobId, bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts(), - indexName, indexPattern); + indexName.get(), indexName.get() + "-*"); for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) { logger.warn("DBQ failure: " + failure); } @@ -280,12 +274,13 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri }, failureHandler); - // Step 6. If we did not delete the index, we run a delete by query + // Step 7. If we did not delete the index, we run a delete by query ActionListener deleteByQueryExecutor = ActionListener.wrap( response -> { if (response) { - logger.info("Running DBQ on [" + indexName + "," + indexPattern + "] for job [" + jobId + "]"); - DeleteByQueryRequest request = new DeleteByQueryRequest(indexName, indexPattern); + String indexPattern = indexName.get() + "-*"; + logger.info("Running DBQ on [" + indexName.get() + "," + indexPattern + "] for job [" + jobId + "]"); + DeleteByQueryRequest request = new DeleteByQueryRequest(indexName.get(), indexPattern); ConstantScoreQueryBuilder query = new ConstantScoreQueryBuilder(new TermQueryBuilder(Job.ID.getPreferredName(), jobId)); request.setQuery(query); @@ -301,15 +296,15 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri }, failureHandler); - // Step 5. If we have any hits, that means we are NOT the only job on this index, and should not delete it + // Step 6. If we have any hits, that means we are NOT the only job on this index, and should not delete it // if we do not have any hits, we can drop the index and then skip the DBQ and alias deletion ActionListener customIndexSearchHandler = ActionListener.wrap( searchResponse -> { if (searchResponse == null || searchResponse.getHits().totalHits > 0) { deleteByQueryExecutor.onResponse(true); // We need to run DBQ and alias deletion } else { - logger.info("Running DELETE Index on [" + indexName + "] for job [" + jobId + "]"); - DeleteIndexRequest request = new DeleteIndexRequest(indexName); + logger.info("Running DELETE Index on [" + indexName.get() + "] for job [" + jobId + "]"); + DeleteIndexRequest request = new DeleteIndexRequest(indexName.get()); request.indicesOptions(IndicesOptions.lenientExpandOpen()); // If we have deleted the index, then we don't need to delete the aliases or run the DBQ executeAsyncWithOrigin( @@ -331,9 +326,11 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri } ); - // Step 4. Determine if we are on a shared index by looking at `.ml-anomalies-shared` or the custom index's aliases - ActionListener deleteCategorizerStateHandler = ActionListener.wrap( - response -> { + // Step 5. Determine if we are on a shared index by looking at `.ml-anomalies-shared` or the custom index's aliases + ActionListener getJobHandler = ActionListener.wrap( + builder -> { + Job job = builder.build(); + indexName.set(job.getResultsIndexName()); if (indexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT)) { //don't bother searching the index any further, we are on the default shared @@ -344,7 +341,7 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri .query(QueryBuilders.boolQuery().filter( QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)))); - SearchRequest searchRequest = new SearchRequest(indexName); + SearchRequest searchRequest = new SearchRequest(indexName.get()); searchRequest.source(source); executeAsyncWithOrigin(parentTaskClient, ML_ORIGIN, SearchAction.INSTANCE, searchRequest, customIndexSearchHandler); } @@ -352,6 +349,14 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri failureHandler ); + // Step 4. Get the job as the result index name is required + ActionListener deleteCategorizerStateHandler = ActionListener.wrap( + response -> { + jobConfigProvider.getJob(jobId, getJobHandler); + }, + failureHandler + ); + // Step 3. Delete quantiles done, delete the categorizer state ActionListener deleteQuantilesHandler = ActionListener.wrap( response -> deleteCategorizerState(parentTaskClient, jobId, 1, deleteCategorizerStateHandler), @@ -554,36 +559,28 @@ public void onFailure(Exception e) { } } - private void markJobAsDeleting(String jobId, ActionListener listener, boolean force) { - clusterService.submitStateUpdateTask("mark-job-as-deleted", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - PersistentTasksCustomMetaData tasks = currentState.metaData().custom(PersistentTasksCustomMetaData.TYPE); - MlMetadata.Builder builder = new MlMetadata.Builder(MlMetadata.getMlMetadata(currentState)); - builder.markJobAsDeleting(jobId, tasks, force); - return buildNewClusterState(currentState, builder); - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - logger.debug("Job [" + jobId + "] is successfully marked as deleted"); - listener.onResponse(true); - } - }); + private void checkJobIsNotOpen(String jobId, ClusterState state) { + PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData.PersistentTask jobTask = MlTasks.getJobTask(jobId, tasks); + if (jobTask != null) { + JobTaskState jobTaskState = (JobTaskState) jobTask.getState(); + throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] because the job is " + + ((jobTaskState == null) ? JobState.OPENING : jobTaskState.getState())); + } } - static boolean jobIsDeletedFromState(String jobId, ClusterState clusterState) { - return !MlMetadata.getMlMetadata(clusterState).getJobs().containsKey(jobId); - } + private void markJobAsDeletingIfNotUsed(String jobId, ActionListener listener) { - private static ClusterState buildNewClusterState(ClusterState currentState, MlMetadata.Builder builder) { - ClusterState.Builder newState = ClusterState.builder(currentState); - newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build()); - return newState.build(); + datafeedConfigProvider.findDatafeedsForJobIds(Collections.singletonList(jobId), ActionListener.wrap( + datafeedIds -> { + if (datafeedIds.isEmpty() == false) { + listener.onFailure(ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] because datafeed [" + + datafeedIds.iterator().next() + "] refers to it")); + return; + } + jobConfigProvider.markJobAsDeleting(jobId, listener); + }, + listener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 26e9ee3019b04..2c19f081956f7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.job.persistence; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -25,6 +26,9 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -40,6 +44,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -64,6 +69,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -202,27 +208,32 @@ public void onFailure(Exception e) { } /** - * Delete the anomaly detector job config document + * Delete the anomaly detector job config document. + * {@code errorIfMissing} controls whether or not an error is returned + * if the document does not exist. * * @param jobId The job id + * @param errorIfMissing If the job document does not exist and this is true + * listener fails with a ResourceNotFoundException else + * the DeleteResponse is always return. * @param actionListener Deleted job listener */ - public void deleteJob(String jobId, ActionListener actionListener) { + public void deleteJob(String jobId, boolean errorIfMissing, ActionListener actionListener) { DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener() { @Override public void onResponse(DeleteResponse deleteResponse) { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - actionListener.onFailure(ExceptionsHelper.missingJobException(jobId)); - return; + if (errorIfMissing) { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + actionListener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED; } - - assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED; actionListener.onResponse(deleteResponse); } - @Override public void onFailure(Exception e) { actionListener.onFailure(e); @@ -423,6 +434,35 @@ public void onFailure(Exception e) { }); } + /** + * Sets the job's {@code deleting} field to true + * @param jobId The job to mark as deleting + * @param listener Responds with true if successful else an error + */ + public void markJobAsDeleting(String jobId, ActionListener listener) { + UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + updateRequest.retryOnConflict(3); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + updateRequest.doc(Collections.singletonMap(Job.DELETING.getPreferredName(), Boolean.TRUE)); + + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateAction.INSTANCE, updateRequest, ActionListener.wrap( + response -> { + assert (response.getResult() == DocWriteResponse.Result.UPDATED) || + (response.getResult() == DocWriteResponse.Result.NOOP); + listener.onResponse(Boolean.TRUE); + }, + e -> { + ElasticsearchException[] causes = ElasticsearchException.guessRootCauses(e); + if (causes[0] instanceof DocumentMissingException) { + listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + } else { + listener.onFailure(e); + } + } + )); + } + /** * Expands an expression into the set of matching names. {@code expresssion} * may be a wildcard, a job group, a job Id or a list of those. diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java deleted file mode 100644 index 7464348adb9aa..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.action; - -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlMetadata; -import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; - -import java.util.Date; - -public class TransportDeleteJobActionTests extends ESTestCase { - - public void testJobIsDeletedFromState() { - MlMetadata mlMetadata = MlMetadata.EMPTY_METADATA; - - ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata)) - .build(); - - assertTrue(TransportDeleteJobAction.jobIsDeletedFromState("job_id_1", clusterState)); - - MlMetadata.Builder mlBuilder = new MlMetadata.Builder(); - mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_1").build(new Date()), false); - mlMetadata = mlBuilder.build(); - clusterState = ClusterState.builder(new ClusterName("_name")) - .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata)) - .build(); - - assertFalse(TransportDeleteJobAction.jobIsDeletedFromState("job_id_1", clusterState)); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index e18edbbbd1b25..d3056680231e3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -80,7 +80,7 @@ public void testValidate_jobMarkedAsDeleting() { jobBuilder.setDeleting(true); Exception e = expectThrows(ElasticsearchStatusException.class, () -> TransportOpenJobAction.validate("job_id", jobBuilder.build())); - assertEquals("Cannot open job [job_id] because it has been marked as deleted", e.getMessage()); + assertEquals("Cannot open job [job_id] because it is being deleted", e.getMessage()); } public void testValidate_jobWithoutVersion() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index 63f67c37dd944..cb4284c874f77 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -158,7 +158,7 @@ public void testCrud() throws InterruptedException { // Delete Job AtomicReference deleteJobResponseHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, actionListener), + blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, true, actionListener), deleteJobResponseHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertThat(deleteJobResponseHolder.get().getResult(), equalTo(DocWriteResponse.Result.DELETED)); @@ -172,10 +172,17 @@ public void testCrud() throws InterruptedException { // Delete deleted job deleteJobResponseHolder.set(null); exceptionHolder.set(null); - blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, actionListener), + blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, true, actionListener), deleteJobResponseHolder, exceptionHolder); assertNull(deleteJobResponseHolder.get()); assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); + + // and again with errorIfMissing set false + deleteJobResponseHolder.set(null); + exceptionHolder.set(null); + blockingCall(actionListener -> jobConfigProvider.deleteJob(jobId, false, actionListener), + deleteJobResponseHolder, exceptionHolder); + assertEquals(DocWriteResponse.Result.NOT_FOUND, deleteJobResponseHolder.get().getResult()); } public void testGetJobs() throws Exception { @@ -482,6 +489,29 @@ public void testValidateDatafeedJob() throws Exception { assertEquals(Messages.DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD, exceptionHolder.get().getMessage()); } + public void testMarkAsDeleting() throws Exception { + AtomicReference responseHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + blockingCall(listener -> jobConfigProvider.markJobAsDeleting("missing-job", listener), responseHolder, exceptionHolder); + assertNull(responseHolder.get()); + assertEquals(ResourceNotFoundException.class, exceptionHolder.get().getClass()); + + String jobId = "mark-as-deleting-job"; + putJob(createJob(jobId, Collections.emptyList())); + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + exceptionHolder.set(null); + blockingCall(listener -> jobConfigProvider.markJobAsDeleting(jobId, listener), responseHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertTrue(responseHolder.get()); + + // repeat the update for good measure + blockingCall(listener -> jobConfigProvider.markJobAsDeleting(jobId, listener), responseHolder, exceptionHolder); + assertTrue(responseHolder.get()); + assertNull(exceptionHolder.get()); + } + private static Job.Builder createJob(String jobId, List groups) { Detector.Builder d1 = new Detector.Builder("info_content", "domain"); d1.setOverFieldName("client"); From 040da1303c44b4a096931ae9fb678432b324e806 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 18 Oct 2018 14:45:33 +0100 Subject: [PATCH 13/57] [ML] Convert job data remover to work with index configs (#34532) --- .../TransportDeleteExpiredDataAction.java | 4 +- .../job/persistence/BatchedJobsIterator.java | 44 +++++ .../AbstractExpiredJobDataRemover.java | 80 +++++++-- .../ExpiredModelSnapshotsRemover.java | 8 +- .../job/retention/ExpiredResultsRemover.java | 5 +- .../AbstractExpiredJobDataRemoverTests.java | 167 ++++++++++++++++++ .../ExpiredModelSnapshotsRemoverTests.java | 95 +++------- .../retention/ExpiredResultsRemoverTests.java | 35 ++-- 8 files changed, 315 insertions(+), 123 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedJobsIterator.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemoverTests.java diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java index 33c992e27c337..6f8370a144b59 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java @@ -57,9 +57,9 @@ protected void doExecute(Task task, DeleteExpiredDataAction.Request request, private void deleteExpiredData(ActionListener listener) { Auditor auditor = new Auditor(client, clusterService.getNodeName()); List dataRemovers = Arrays.asList( - new ExpiredResultsRemover(client, clusterService, auditor), + new ExpiredResultsRemover(client, auditor), new ExpiredForecastsRemover(client, threadPool), - new ExpiredModelSnapshotsRemover(client, threadPool, clusterService), + new ExpiredModelSnapshotsRemover(client, threadPool), new UnusedStateRemover(client, clusterService) ); Iterator dataRemoversIterator = new VolatileCursorIterator<>(dataRemovers); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedJobsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedJobsIterator.java new file mode 100644 index 0000000000000..e274b720e701f --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedJobsIterator.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.core.ml.job.config.Job; + +import java.io.IOException; +import java.io.InputStream; + +public class BatchedJobsIterator extends BatchedDocumentsIterator { + + public BatchedJobsIterator(Client client, String index) { + super(client, index); + } + + @Override + protected QueryBuilder getQuery() { + return new TermQueryBuilder(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE); + } + + @Override + protected Job.Builder map(SearchHit hit) { + try (InputStream stream = hit.getSourceRef().streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + return Job.LENIENT_PARSER.apply(parser, null); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse job document [" + hit.getId() + "]", e); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemover.java index 8364e015a3456..b595c564ab9aa 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemover.java @@ -6,23 +6,23 @@ package org.elasticsearch.xpack.ml.job.retention; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.results.Result; +import org.elasticsearch.xpack.ml.job.persistence.BatchedJobsIterator; import org.elasticsearch.xpack.ml.utils.VolatileCursorIterator; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; -import java.util.ArrayList; +import java.util.Deque; import java.util.Iterator; import java.util.List; -import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Removes job data that expired with respect to their retention period. @@ -33,10 +33,10 @@ */ abstract class AbstractExpiredJobDataRemover implements MlDataRemover { - private final ClusterService clusterService; + private final Client client; - AbstractExpiredJobDataRemover(ClusterService clusterService) { - this.clusterService = Objects.requireNonNull(clusterService); + AbstractExpiredJobDataRemover(Client client) { + this.client = client; } @Override @@ -44,12 +44,18 @@ public void remove(ActionListener listener) { removeData(newJobIterator(), listener); } - private void removeData(Iterator jobIterator, ActionListener listener) { + private void removeData(WrappedBatchedJobsIterator jobIterator, ActionListener listener) { if (jobIterator.hasNext() == false) { listener.onResponse(true); return; } Job job = jobIterator.next(); + if (job == null) { + // maybe null if the batched iterator search return no results + listener.onResponse(true); + return; + } + Long retentionDays = getRetentionDays(job); if (retentionDays == null) { removeData(jobIterator, listener); @@ -59,14 +65,9 @@ private void removeData(Iterator jobIterator, ActionListener liste removeDataBefore(job, cutoffEpochMs, ActionListener.wrap(response -> removeData(jobIterator, listener), listener::onFailure)); } - private Iterator newJobIterator() { - ClusterState clusterState = clusterService.state(); - List jobs = new ArrayList<>(MlMetadata.getMlMetadata(clusterState).getJobs().values()); - return createVolatileCursorIterator(jobs); - } - - protected static Iterator createVolatileCursorIterator(List items) { - return new VolatileCursorIterator(items); + private WrappedBatchedJobsIterator newJobIterator() { + BatchedJobsIterator jobsIterator = new BatchedJobsIterator(client, AnomalyDetectorsIndex.configIndexName()); + return new WrappedBatchedJobsIterator(jobsIterator); } private long calcCutoffEpochMs(long retentionDays) { @@ -87,4 +88,49 @@ protected static BoolQueryBuilder createQuery(String jobId, long cutoffEpochMs) .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) .filter(QueryBuilders.rangeQuery(Result.TIMESTAMP.getPreferredName()).lt(cutoffEpochMs).format("epoch_millis")); } + + /** + * BatchedJobsIterator efficiently returns batches of jobs using a scroll + * search but AbstractExpiredJobDataRemover works with one job at a time. + * This class abstracts away the logic of pulling one job at a time from + * multiple batches. + */ + private class WrappedBatchedJobsIterator implements Iterator { + private final BatchedJobsIterator batchedIterator; + private VolatileCursorIterator currentBatch; + + WrappedBatchedJobsIterator(BatchedJobsIterator batchedIterator) { + this.batchedIterator = batchedIterator; + } + + @Override + public boolean hasNext() { + return (currentBatch != null && currentBatch.hasNext()) || batchedIterator.hasNext(); + } + + /** + * Before BatchedJobsIterator has run a search it reports hasNext == true + * but the first search may return no results. In that case null is return + * and clients have to handle null. + */ + @Override + public Job next() { + if (currentBatch != null && currentBatch.hasNext()) { + return currentBatch.next(); + } + + // currentBatch is either null or all its elements have been iterated. + // get the next currentBatch + currentBatch = createBatchIteratorFromBatch(batchedIterator.next()); + + // BatchedJobsIterator.hasNext maybe true if searching the first time + // but no results are returned. + return currentBatch.hasNext() ? currentBatch.next() : null; + } + + private VolatileCursorIterator createBatchIteratorFromBatch(Deque builders) { + List jobs = builders.stream().map(Job.Builder::build).collect(Collectors.toList()); + return new VolatileCursorIterator<>(jobs); + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java index 47a10a8aea381..2272121a68ff3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -27,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.utils.VolatileCursorIterator; import java.util.ArrayList; import java.util.Iterator; @@ -57,8 +57,8 @@ public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover private final Client client; private final ThreadPool threadPool; - public ExpiredModelSnapshotsRemover(Client client, ThreadPool threadPool, ClusterService clusterService) { - super(clusterService); + public ExpiredModelSnapshotsRemover(Client client, ThreadPool threadPool) { + super(client); this.client = Objects.requireNonNull(client); this.threadPool = Objects.requireNonNull(threadPool); } @@ -103,7 +103,7 @@ public void onResponse(SearchResponse searchResponse) { for (SearchHit hit : searchResponse.getHits()) { modelSnapshots.add(ModelSnapshot.fromJson(hit.getSourceRef())); } - deleteModelSnapshots(createVolatileCursorIterator(modelSnapshots), listener); + deleteModelSnapshots(new VolatileCursorIterator<>(modelSnapshots), listener); } catch (Exception e) { onFailure(e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java index c882c90116880..dfa9d66814ebb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java @@ -9,7 +9,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -48,8 +47,8 @@ public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover { private final Client client; private final Auditor auditor; - public ExpiredResultsRemover(Client client, ClusterService clusterService, Auditor auditor) { - super(clusterService); + public ExpiredResultsRemover(Client client, Auditor auditor) { + super(client); this.client = Objects.requireNonNull(client); this.auditor = Objects.requireNonNull(auditor); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemoverTests.java new file mode 100644 index 0000000000000..c2318d1cb4664 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/AbstractExpiredJobDataRemoverTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.retention; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AbstractExpiredJobDataRemoverTests extends ESTestCase { + + // We can't test an abstract class so make a concrete class + // as simple as possible + private class ConcreteExpiredJobDataRemover extends AbstractExpiredJobDataRemover { + + private int getRetentionDaysCallCount = 0; + + ConcreteExpiredJobDataRemover(Client client) { + super(client); + } + + @Override + protected Long getRetentionDays(Job job) { + getRetentionDaysCallCount++; + // cover both code paths + return randomBoolean() ? null : 0L; + } + + @Override + protected void removeDataBefore(Job job, long cutoffEpochMs, ActionListener listener) { + listener.onResponse(Boolean.TRUE); + } + } + + private Client client; + + @Before + public void setUpTests() { + client = mock(Client.class); + } + + static SearchResponse createSearchResponse(List toXContents) throws IOException { + return createSearchResponse(toXContents, toXContents.size()); + } + + private static SearchResponse createSearchResponse(List toXContents, int totalHits) throws IOException { + SearchHit[] hitsArray = new SearchHit[toXContents.size()]; + for (int i = 0; i < toXContents.size(); i++) { + hitsArray[i] = new SearchHit(randomInt()); + XContentBuilder jsonBuilder = JsonXContent.contentBuilder(); + toXContents.get(i).toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); + hitsArray[i].sourceRef(BytesReference.bytes(jsonBuilder)); + } + SearchHits hits = new SearchHits(hitsArray, totalHits, 1.0f); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn(hits); + return searchResponse; + } + + public void testRemoveGivenNoJobs() throws IOException { + SearchResponse response = createSearchResponse(Collections.emptyList()); + + ActionFuture future = mock(ActionFuture.class); + when(future.actionGet()).thenReturn(response); + when(client.search(any())).thenReturn(future); + + TestListener listener = new TestListener(); + ConcreteExpiredJobDataRemover remover = new ConcreteExpiredJobDataRemover(client); + remover.remove(listener); + + listener.waitToCompletion(); + assertThat(listener.success, is(true)); + assertEquals(remover.getRetentionDaysCallCount, 0); + } + + + public void testRemoveGivenMulipleBatches() throws IOException { + // This is testing AbstractExpiredJobDataRemover.WrappedBatchedJobsIterator + int totalHits = 7; + List responses = new ArrayList<>(); + responses.add(createSearchResponse(Arrays.asList( + JobTests.buildJobBuilder("job1").build(), + JobTests.buildJobBuilder("job2").build(), + JobTests.buildJobBuilder("job3").build() + ), totalHits)); + + responses.add(createSearchResponse(Arrays.asList( + JobTests.buildJobBuilder("job4").build(), + JobTests.buildJobBuilder("job5").build(), + JobTests.buildJobBuilder("job6").build() + ), totalHits)); + + responses.add(createSearchResponse(Collections.singletonList( + JobTests.buildJobBuilder("job7").build() + ), totalHits)); + + + AtomicInteger searchCount = new AtomicInteger(0); + + ActionFuture future = mock(ActionFuture.class); + doAnswer(invocationOnMock -> responses.get(searchCount.getAndIncrement())).when(future).actionGet(); + when(client.search(any())).thenReturn(future); + + TestListener listener = new TestListener(); + ConcreteExpiredJobDataRemover remover = new ConcreteExpiredJobDataRemover(client); + remover.remove(listener); + + listener.waitToCompletion(); + assertThat(listener.success, is(true)); + assertEquals(searchCount.get(), 3); + assertEquals(remover.getRetentionDaysCallCount, 7); + } + + static class TestListener implements ActionListener { + + boolean success; + private final CountDownLatch latch = new CountDownLatch(1); + + @Override + public void onResponse(Boolean aBoolean) { + success = aBoolean; + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + latch.countDown(); + } + + public void waitToCompletion() { + try { + latch.await(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + fail("listener timed out before completing"); + } + } + } + +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java index 9f056e91854c3..02d747fb80a50 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java @@ -5,27 +5,18 @@ */ package org.elasticsearch.xpack.ml.job.retention; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.mock.orig.Mockito; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobTests; @@ -40,26 +31,22 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.ml.job.retention.AbstractExpiredJobDataRemoverTests.TestListener; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; import static org.mockito.Matchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ExpiredModelSnapshotsRemoverTests extends ESTestCase { private Client client; private ThreadPool threadPool; - private ClusterService clusterService; - private ClusterState clusterState; private List capturedSearchRequests; private List capturedDeleteModelSnapshotRequests; private List searchResponsesPerCall; @@ -70,9 +57,6 @@ public void setUpTests() { capturedSearchRequests = new ArrayList<>(); capturedDeleteModelSnapshotRequests = new ArrayList<>(); searchResponsesPerCall = new ArrayList<>(); - clusterService = mock(ClusterService.class); - clusterState = mock(ClusterState.class); - when(clusterService.state()).thenReturn(clusterState); client = mock(Client.class); listener = new TestListener(); @@ -89,7 +73,7 @@ public void shutdownThreadPool() throws InterruptedException { terminate(threadPool); } - public void testRemove_GivenJobsWithoutRetentionPolicy() { + public void testRemove_GivenJobsWithoutRetentionPolicy() throws IOException { givenClientRequestsSucceed(); givenJobs(Arrays.asList( JobTests.buildJobBuilder("foo").build(), @@ -100,10 +84,11 @@ public void testRemove_GivenJobsWithoutRetentionPolicy() { listener.waitToCompletion(); assertThat(listener.success, is(true)); + verify(client).search(any()); Mockito.verifyNoMoreInteractions(client); } - public void testRemove_GivenJobWithoutActiveSnapshot() { + public void testRemove_GivenJobWithoutActiveSnapshot() throws IOException { givenClientRequestsSucceed(); givenJobs(Arrays.asList(JobTests.buildJobBuilder("foo").setModelSnapshotRetentionDays(7L).build())); @@ -111,6 +96,7 @@ public void testRemove_GivenJobWithoutActiveSnapshot() { listener.waitToCompletion(); assertThat(listener.success, is(true)); + verify(client).search(any()); Mockito.verifyNoMoreInteractions(client); } @@ -125,8 +111,8 @@ public void testRemove_GivenJobsWithMixedRetentionPolicies() throws IOException List snapshots1JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-1", "snapshots-1_1"), createModelSnapshot("snapshots-1", "snapshots-1_2")); List snapshots2JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-2", "snapshots-2_1")); - searchResponsesPerCall.add(createSearchResponse(snapshots1JobSnapshots)); - searchResponsesPerCall.add(createSearchResponse(snapshots2JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots1JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots2JobSnapshots)); createExpiredModelSnapshotsRemover().remove(listener); @@ -162,8 +148,8 @@ public void testRemove_GivenClientSearchRequestsFail() throws IOException { List snapshots1JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-1", "snapshots-1_1"), createModelSnapshot("snapshots-1", "snapshots-1_2")); List snapshots2JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-2", "snapshots-2_1")); - searchResponsesPerCall.add(createSearchResponse(snapshots1JobSnapshots)); - searchResponsesPerCall.add(createSearchResponse(snapshots2JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots1JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots2JobSnapshots)); createExpiredModelSnapshotsRemover().remove(listener); @@ -188,8 +174,8 @@ public void testRemove_GivenClientDeleteSnapshotRequestsFail() throws IOExceptio List snapshots1JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-1", "snapshots-1_1"), createModelSnapshot("snapshots-1", "snapshots-1_2")); List snapshots2JobSnapshots = Arrays.asList(createModelSnapshot("snapshots-2", "snapshots-2_1")); - searchResponsesPerCall.add(createSearchResponse(snapshots1JobSnapshots)); - searchResponsesPerCall.add(createSearchResponse(snapshots2JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots1JobSnapshots)); + searchResponsesPerCall.add(AbstractExpiredJobDataRemoverTests.createSearchResponse(snapshots2JobSnapshots)); createExpiredModelSnapshotsRemover().remove(listener); @@ -206,38 +192,22 @@ public void testRemove_GivenClientDeleteSnapshotRequestsFail() throws IOExceptio assertThat(deleteSnapshotRequest.getSnapshotId(), equalTo("snapshots-1_1")); } - private void givenJobs(List jobs) { - Map jobsMap = new HashMap<>(); - jobs.stream().forEach(job -> jobsMap.put(job.getId(), job)); - MlMetadata mlMetadata = mock(MlMetadata.class); - when(mlMetadata.getJobs()).thenReturn(jobsMap); - MetaData metadata = mock(MetaData.class); - when(metadata.custom(MlMetadata.TYPE)).thenReturn(mlMetadata); - when(clusterState.getMetaData()).thenReturn(metadata); + private void givenJobs(List jobs) throws IOException { + SearchResponse response = AbstractExpiredJobDataRemoverTests.createSearchResponse(jobs); + + ActionFuture future = mock(ActionFuture.class); + when(future.actionGet()).thenReturn(response); + when(client.search(any())).thenReturn(future); } private ExpiredModelSnapshotsRemover createExpiredModelSnapshotsRemover() { - return new ExpiredModelSnapshotsRemover(client, threadPool, clusterService); + return new ExpiredModelSnapshotsRemover(client, threadPool); } private static ModelSnapshot createModelSnapshot(String jobId, String snapshotId) { return new ModelSnapshot.Builder(jobId).setSnapshotId(snapshotId).build(); } - private static SearchResponse createSearchResponse(List modelSnapshots) throws IOException { - SearchHit[] hitsArray = new SearchHit[modelSnapshots.size()]; - for (int i = 0; i < modelSnapshots.size(); i++) { - hitsArray[i] = new SearchHit(randomInt()); - XContentBuilder jsonBuilder = JsonXContent.contentBuilder(); - modelSnapshots.get(i).toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); - hitsArray[i].sourceRef(BytesReference.bytes(jsonBuilder)); - } - SearchHits hits = new SearchHits(hitsArray, hitsArray.length, 1.0f); - SearchResponse searchResponse = mock(SearchResponse.class); - when(searchResponse.getHits()).thenReturn(hits); - return searchResponse; - } - private void givenClientRequestsSucceed() { givenClientRequests(true, true); } @@ -283,29 +253,4 @@ public Void answer(InvocationOnMock invocationOnMock) { }).when(client).execute(same(DeleteModelSnapshotAction.INSTANCE), any(), any()); } - private class TestListener implements ActionListener { - - private boolean success; - private final CountDownLatch latch = new CountDownLatch(1); - - @Override - public void onResponse(Boolean aBoolean) { - success = aBoolean; - latch.countDown(); - } - - @Override - public void onFailure(Exception e) { - latch.countDown(); - } - - public void waitToCompletion() { - try { - latch.await(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - fail("listener timed out before completing"); - } - } - } - } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java index af9ec8b84a6bd..7dc258a322ac3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java @@ -5,11 +5,10 @@ */ package org.elasticsearch.xpack.ml.job.retention; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -18,7 +17,6 @@ import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobTests; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -31,9 +29,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; @@ -46,17 +42,12 @@ public class ExpiredResultsRemoverTests extends ESTestCase { private Client client; - private ClusterService clusterService; - private ClusterState clusterState; private List capturedDeleteByQueryRequests; private ActionListener listener; @Before public void setUpTests() { capturedDeleteByQueryRequests = new ArrayList<>(); - clusterService = mock(ClusterService.class); - clusterState = mock(ClusterState.class); - when(clusterService.state()).thenReturn(clusterState); client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); when(client.threadPool()).thenReturn(threadPool); @@ -74,17 +65,18 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { listener = mock(ActionListener.class); } - public void testRemove_GivenNoJobs() { + public void testRemove_GivenNoJobs() throws IOException { givenClientRequestsSucceed(); givenJobs(Collections.emptyList()); createExpiredResultsRemover().remove(listener); verify(listener).onResponse(true); + verify(client).search(any()); Mockito.verifyNoMoreInteractions(client); } - public void testRemove_GivenJobsWithoutRetentionPolicy() { + public void testRemove_GivenJobsWithoutRetentionPolicy() throws IOException { givenClientRequestsSucceed(); givenJobs(Arrays.asList( JobTests.buildJobBuilder("foo").build(), @@ -94,6 +86,7 @@ public void testRemove_GivenJobsWithoutRetentionPolicy() { createExpiredResultsRemover().remove(listener); verify(listener).onResponse(true); + verify(client).search(any()); Mockito.verifyNoMoreInteractions(client); } @@ -158,17 +151,15 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { }).when(client).execute(same(DeleteByQueryAction.INSTANCE), any(), any()); } - private void givenJobs(List jobs) { - Map jobsMap = new HashMap<>(); - jobs.stream().forEach(job -> jobsMap.put(job.getId(), job)); - MlMetadata mlMetadata = mock(MlMetadata.class); - when(mlMetadata.getJobs()).thenReturn(jobsMap); - MetaData metadata = mock(MetaData.class); - when(metadata.custom(MlMetadata.TYPE)).thenReturn(mlMetadata); - when(clusterState.getMetaData()).thenReturn(metadata); + private void givenJobs(List jobs) throws IOException { + SearchResponse response = AbstractExpiredJobDataRemoverTests.createSearchResponse(jobs); + + ActionFuture future = mock(ActionFuture.class); + when(future.actionGet()).thenReturn(response); + when(client.search(any())).thenReturn(future); } private ExpiredResultsRemover createExpiredResultsRemover() { - return new ExpiredResultsRemover(client, clusterService, mock(Auditor.class)); + return new ExpiredResultsRemover(client, mock(Auditor.class)); } } \ No newline at end of file From e8c3951a715371b9a1f5d5292ad72c3f3a289a6a Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 22 Oct 2018 10:07:24 +0100 Subject: [PATCH 14/57] [ML] Job in index: Get datafeed and job stats from index (#34645) --- .../ml/action/TransportCloseJobAction.java | 2 +- .../TransportGetDatafeedsStatsAction.java | 33 ++++---- .../action/TransportGetJobsStatsAction.java | 55 ++++++++------ .../xpack/ml/job/JobManager.java | 2 +- .../ml/job/persistence/JobConfigProvider.java | 31 +++++--- .../action/TransportCloseJobActionTests.java | 4 +- .../TransportGetJobsStatsActionTests.java | 34 ++------- .../ml/integration/JobConfigProviderIT.java | 76 +++++++++++++------ 8 files changed, 136 insertions(+), 101 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index e6cc8b69e1bd1..cdd8c636b78cf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -107,7 +107,7 @@ protected void doExecute(Task task, CloseJobAction.Request request, ActionListen */ PersistentTasksCustomMetaData tasksMetaData = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( + jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), true, ActionListener.wrap( expandedJobIds -> { validate(expandedJobIds, request.isForce(), tasksMetaData, ActionListener.wrap( response -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java index 1f7e55fc488a6..f77b7b14de236 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java @@ -16,29 +16,32 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; public class TransportGetDatafeedsStatsAction extends TransportMasterNodeReadAction { + private final DatafeedConfigProvider datafeedConfigProvider; + @Inject public TransportGetDatafeedsStatsAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { + IndexNameExpressionResolver indexNameExpressionResolver, + DatafeedConfigProvider datafeedConfigProvider) { super(settings, GetDatafeedsStatsAction.NAME, transportService, clusterService, threadPool, actionFilters, GetDatafeedsStatsAction.Request::new, indexNameExpressionResolver); + this.datafeedConfigProvider = datafeedConfigProvider; } @Override @@ -56,16 +59,18 @@ protected void masterOperation(GetDatafeedsStatsAction.Request request, ClusterS ActionListener listener) throws Exception { logger.debug("Get stats for datafeed '{}'", request.getDatafeedId()); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); - Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds()); - - PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - List results = expandedDatafeedIds.stream() - .map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress)) - .collect(Collectors.toList()); - QueryPage statsPage = new QueryPage<>(results, results.size(), - DatafeedConfig.RESULTS_FIELD); - listener.onResponse(new GetDatafeedsStatsAction.Response(statsPage)); + datafeedConfigProvider.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds(), ActionListener.wrap( + expandedDatafeedIds -> { + PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + List results = expandedDatafeedIds.stream() + .map(datafeedId -> getDatafeedStats(datafeedId, state, tasksInProgress)) + .collect(Collectors.toList()); + QueryPage statsPage = new QueryPage<>(results, results.size(), + DatafeedConfig.RESULTS_FIELD); + listener.onResponse(new GetDatafeedsStatsAction.Response(statsPage)); + }, + listener::onFailure + )); } private static GetDatafeedsStatsAction.Response.DatafeedStats getDatafeedStats(String datafeedId, ClusterState state, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java index 7217fcc6ec9a7..3819a15843fbd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java @@ -19,10 +19,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.stats.ForecastStats; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -53,27 +53,35 @@ public class TransportGetJobsStatsAction extends TransportTasksAction listener) { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterService.state()); - request.setExpandedJobsIds(new ArrayList<>(mlMetadata.expandJobIds(request.getJobId(), request.allowNoJobs()))); - ActionListener finalListener = listener; - listener = ActionListener.wrap(response -> gatherStatsForClosedJobs(mlMetadata, - request, response, finalListener), listener::onFailure); - super.doExecute(task, request, listener); + protected void doExecute(Task task, GetJobsStatsAction.Request request, ActionListener finalListener) { + + jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), true, ActionListener.wrap( + expandedIds -> { + request.setExpandedJobsIds(new ArrayList<>(expandedIds)); + ActionListener jobStatsListener = ActionListener.wrap( + response -> gatherStatsForClosedJobs(request, response, finalListener), + finalListener::onFailure + ); + super.doExecute(task, request, jobStatsListener); + }, + finalListener::onFailure + )); } @Override @@ -121,21 +129,20 @@ protected void taskOperation(GetJobsStatsAction.Request request, TransportOpenJo // Up until now we gathered the stats for jobs that were open, // This method will fetch the stats for missing jobs, that was stored in the jobs index - void gatherStatsForClosedJobs(MlMetadata mlMetadata, GetJobsStatsAction.Request request, GetJobsStatsAction.Response response, + void gatherStatsForClosedJobs(GetJobsStatsAction.Request request, GetJobsStatsAction.Response response, ActionListener listener) { - List jobIds = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - request.getExpandedJobsIds(), response.getResponse().results()); - if (jobIds.isEmpty()) { + List closedJobIds = determineJobIdsWithoutLiveStats(request.getExpandedJobsIds(), response.getResponse().results()); + if (closedJobIds.isEmpty()) { listener.onResponse(response); return; } - AtomicInteger counter = new AtomicInteger(jobIds.size()); - AtomicArray jobStats = new AtomicArray<>(jobIds.size()); + AtomicInteger counter = new AtomicInteger(closedJobIds.size()); + AtomicArray jobStats = new AtomicArray<>(closedJobIds.size()); PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - for (int i = 0; i < jobIds.size(); i++) { + for (int i = 0; i < closedJobIds.size(); i++) { int slot = i; - String jobId = jobIds.get(i); + String jobId = closedJobIds.get(i); gatherForecastStats(jobId, forecastStats -> { gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> { JobState jobState = MlTasks.getJobState(jobId, tasks); @@ -178,11 +185,9 @@ static TimeValue durationToTimeValue(Optional duration) { } } - static List determineNonDeletedJobIdsWithoutLiveStats(MlMetadata mlMetadata, - List requestedJobIds, - List stats) { + static List determineJobIdsWithoutLiveStats(List requestedJobIds, + List stats) { Set excludeJobIds = stats.stream().map(GetJobsStatsAction.Response.JobStats::getJobId).collect(Collectors.toSet()); - return requestedJobIds.stream().filter(jobId -> !excludeJobIds.contains(jobId) && - !mlMetadata.isJobDeleting(jobId)).collect(Collectors.toList()); + return requestedJobIds.stream().filter(jobId -> !excludeJobIds.contains(jobId)).collect(Collectors.toList()); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 0393f65bcaba7..80612d6b51402 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -172,7 +172,7 @@ private void getJobFromClusterState(String jobId, ActionListener jobListene public void expandJobs(String expression, boolean allowNoJobs, ActionListener> jobsListener) { Map clusterStateJobs = expandJobsFromClusterState(expression, allowNoJobs, clusterService.state()); - jobConfigProvider.expandJobs(expression, allowNoJobs, ActionListener.wrap( + jobConfigProvider.expandJobs(expression, allowNoJobs, false, ActionListener.wrap( jobBuilders -> { // Check for duplicate jobs for (Job.Builder jb : jobBuilders) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 2c19f081956f7..8ca0c5ff531a0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; @@ -489,11 +490,12 @@ public void markJobAsDeleting(String jobId, ActionListener listener) { * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception + * @param excludeDeleting If true exclude jobs marked as deleting * @param listener The expanded job Ids listener */ - public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener> listener) { + public void expandJobsIds(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); sourceBuilder.docValueField(Job.ID.getPreferredName()); @@ -535,21 +537,22 @@ public void expandJobsIds(String expression, boolean allowNoJobs, ActionListener } /** - * The same logic as {@link #expandJobsIds(String, boolean, ActionListener)} but + * The same logic as {@link #expandJobsIds(String, boolean, boolean, ActionListener)} but * the full anomaly detector job configuration is returned. * - * See {@link #expandJobsIds(String, boolean, ActionListener)} + * See {@link #expandJobsIds(String, boolean, boolean, ActionListener)} * * @param expression the expression to resolve * @param allowNoJobs if {@code false}, an error is thrown when no name matches the {@code expression}. * This only applies to wild card expressions, if {@code expression} is not a * wildcard then setting this true will not suppress the exception + * @param excludeDeleting If true exclude jobs marked as deleting * @param listener The expanded jobs listener */ // NORELEASE jobs should be paged or have a mechanism to return all jobs if there are many of them - public void expandJobs(String expression, boolean allowNoJobs, ActionListener> listener) { + public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) @@ -594,7 +597,7 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener
  • terms = new ArrayList<>(); for (String token : tokens) { if (Regex.isSimpleMatchPattern(token)) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index b7d73b79829ce..2348bc1849932 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -288,11 +288,11 @@ private void mockDatafeedConfigFindDatafeeds(Set datafeedIds) { private void mockJobConfigProviderExpandIds(Set expandedIds) { doAnswer(invocation -> { - ActionListener> listener = (ActionListener>) invocation.getArguments()[2]; + ActionListener> listener = (ActionListener>) invocation.getArguments()[3]; listener.onResponse(expandedIds); return null; - }).when(jobConfigProvider).expandJobsIds(any(), anyBoolean(), any(ActionListener.class)); + }).when(jobConfigProvider).expandJobsIds(any(), anyBoolean(), anyBoolean(), any(ActionListener.class)); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java index 6d4b008570c72..2ee184ec877ed 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; @@ -18,37 +17,27 @@ import java.util.List; import java.util.Optional; -import static org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction.determineNonDeletedJobIdsWithoutLiveStats; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction.determineJobIdsWithoutLiveStats; public class TransportGetJobsStatsActionTests extends ESTestCase { public void testDetermineJobIds() { - MlMetadata mlMetadata = mock(MlMetadata.class); - when(mlMetadata.isJobDeleting(eq("id4"))).thenReturn(true); - - List result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Collections.singletonList("id1"), Collections.emptyList()); + List result = determineJobIdsWithoutLiveStats(Collections.singletonList("id1"), Collections.emptyList()); assertEquals(1, result.size()); assertEquals("id1", result.get(0)); - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Collections.singletonList("id1"), Collections.singletonList( + result = determineJobIdsWithoutLiveStats(Collections.singletonList("id1"), Collections.singletonList( new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null))); assertEquals(0, result.size()); - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Arrays.asList("id1", "id2", "id3"), Collections.emptyList()); + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Collections.emptyList()); assertEquals(3, result.size()); assertEquals("id1", result.get(0)); assertEquals("id2", result.get(1)); assertEquals("id3", result.get(2)); - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Arrays.asList("id1", "id2", "id3"), + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Collections.singletonList(new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.CLOSED, null, null, null)) ); @@ -56,27 +45,18 @@ public void testDetermineJobIds() { assertEquals("id2", result.get(0)); assertEquals("id3", result.get(1)); - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Arrays.asList("id1", "id2", "id3"), Arrays.asList( + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Arrays.asList( new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null), new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null) )); assertEquals(1, result.size()); assertEquals("id2", result.get(0)); - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, Arrays.asList("id1", "id2", "id3"), Arrays.asList( + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Arrays.asList( new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null), new GetJobsStatsAction.Response.JobStats("id2", new DataCounts("id2"), null, null, JobState.OPENED, null, null, null), new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null))); assertEquals(0, result.size()); - - // No jobs running, but job 4 is being deleted - result = determineNonDeletedJobIdsWithoutLiveStats(mlMetadata, - Arrays.asList("id1", "id2", "id3", "id4"), Collections.emptyList()); - assertEquals(3, result.size()); - assertEquals("id1", result.get(0)); - assertEquals("id2", result.get(1)); - assertEquals("id3", result.get(2)); } public void testDurationToTimeValue() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index cb4284c874f77..75198a3350dc5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -273,7 +273,7 @@ public void testAllowNoJobs() throws InterruptedException { AtomicReference> jobIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", false, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", false, true, actionListener), jobIdsHolder, exceptionHolder); assertNull(jobIdsHolder.get()); @@ -282,13 +282,13 @@ public void testAllowNoJobs() throws InterruptedException { assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); exceptionHolder.set(null); - blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, false, actionListener), jobIdsHolder, exceptionHolder); assertNotNull(jobIdsHolder.get()); assertNull(exceptionHolder.get()); AtomicReference> jobsHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.expandJobs("*", false, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", false, true, actionListener), jobsHolder, exceptionHolder); assertNull(jobsHolder.get()); @@ -297,7 +297,7 @@ public void testAllowNoJobs() throws InterruptedException { assertThat(exceptionHolder.get().getMessage(), containsString("No known job with id")); exceptionHolder.set(null); - blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, true, actionListener), jobsHolder, exceptionHolder); assertNotNull(jobsHolder.get()); assertNull(exceptionHolder.get()); @@ -312,21 +312,21 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); // Job Ids - Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, actionListener)); + Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, false, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("tom", "dick", "harry", "harry-jnr")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("tom", "dick", "harry", "harry-jnr")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,harry", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,harry", true, false, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("tom", "harry")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("harry-group,tom", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("harry-group,tom", true, false, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("harry", "harry-jnr", "tom")), expandedIds); AtomicReference exceptionHolder = new AtomicReference<>(); AtomicReference> jobIdsHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,missing1,missing2", true, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,missing1,missing2", true, false, actionListener), jobIdsHolder, exceptionHolder); assertNull(jobIdsHolder.get()); assertNotNull(exceptionHolder.get()); @@ -335,27 +335,27 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { // Job builders List expandedJobsBuilders = blockingCall(actionListener -> - jobConfigProvider.expandJobs("harry-group,tom", false, actionListener)); + jobConfigProvider.expandJobs("harry-group,tom", false, true, actionListener)); List expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(harry, harryJnr, tom)); expandedJobsBuilders = blockingCall(actionListener -> - jobConfigProvider.expandJobs("_all", false, actionListener)); + jobConfigProvider.expandJobs("_all", false, true, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(tom, dick, harry, harryJnr)); expandedJobsBuilders = blockingCall(actionListener -> - jobConfigProvider.expandJobs("tom,harry", false, actionListener)); + jobConfigProvider.expandJobs("tom,harry", false, false, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(tom, harry)); expandedJobsBuilders = blockingCall(actionListener -> - jobConfigProvider.expandJobs("", false, actionListener)); + jobConfigProvider.expandJobs("", false, false, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(tom, dick, harry, harryJnr)); AtomicReference> jobsHolder = new AtomicReference<>(); - blockingCall(actionListener -> jobConfigProvider.expandJobs("tom,missing1,missing2", false, actionListener), + blockingCall(actionListener -> jobConfigProvider.expandJobs("tom,missing1,missing2", false, true, actionListener), jobsHolder, exceptionHolder); assertNull(jobsHolder.get()); assertNotNull(exceptionHolder.get()); @@ -373,36 +373,68 @@ public void testExpandJobs_WildCardExpansion() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); // Test job IDs only - Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, actionListener)); + Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*-1", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*-1", true, true,actionListener)); assertEquals(new TreeSet<>(Arrays.asList("bar-1", "foo-1")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("bar*", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("bar*", true, true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("bar-1", "bar-2", "nbar")), expandedIds); - expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("b*r-1", true, actionListener)); + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("b*r-1", true, true, actionListener)); assertEquals(new TreeSet<>(Collections.singletonList("bar-1")), expandedIds); // Test full job config - List expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("foo*", true, actionListener)); + List expandedJobsBuilders = + blockingCall(actionListener -> jobConfigProvider.expandJobs("foo*", true, true, actionListener)); List expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(foo1, foo2)); - expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("*-1", true, actionListener)); + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("*-1", true, true, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(foo1, bar1)); - expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("bar*", true, actionListener)); + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("bar*", true, true, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(bar1, bar2, nbar)); - expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("b*r-1", true, actionListener)); + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("b*r-1", true, true, actionListener)); expandedJobs = expandedJobsBuilders.stream().map(Job.Builder::build).collect(Collectors.toList()); assertThat(expandedJobs, containsInAnyOrder(bar1)); } + public void testExpandJobIds_excludeDeleting() throws Exception { + putJob(createJob("foo-1", null)); + putJob(createJob("foo-2", null)); + putJob(createJob("foo-deleting", null)); + putJob(createJob("bar", null)); + + Boolean marked = blockingCall(actionListener -> jobConfigProvider.markJobAsDeleting("foo-deleting", actionListener)); + assertTrue(marked); + + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + + Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, false, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2", "foo-deleting")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, true, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2", "bar")), expandedIds); + + expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, false, actionListener)); + assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2", "foo-deleting", "bar")), expandedIds); + + List expandedJobsBuilders = + blockingCall(actionListener -> jobConfigProvider.expandJobs("foo*", true, true, actionListener)); + assertThat(expandedJobsBuilders, hasSize(2)); + + expandedJobsBuilders = blockingCall(actionListener -> jobConfigProvider.expandJobs("foo*", true, false, actionListener)); + assertThat(expandedJobsBuilders, hasSize(3)); + } + public void testExpandGroups() throws Exception { putJob(createJob("apples", Collections.singletonList("fruit"))); putJob(createJob("pears", Collections.singletonList("fruit"))); From 7fd4b9d33d5db8b705be9b428634a3d90ae98115 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 23 Oct 2018 16:40:58 +0100 Subject: [PATCH 15/57] [ML] Job in Index: Convert get calendar events to index docs (#34710) --- .../elasticsearch/xpack/core/ml/MlTasks.java | 9 +++- .../xpack/core/ml/MlTasksTests.java | 8 +++ .../TransportGetCalendarEventsAction.java | 51 +++++++++---------- .../ml/job/persistence/JobConfigProvider.java | 35 +++++++++++++ .../xpack/test/rest/XPackRestIT.java | 3 +- 5 files changed, 77 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index 46685001153d7..5ba04fcc4087c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -83,10 +84,14 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis * All anomaly detector jobs are returned regardless of the status of the * task (OPEN, CLOSED, FAILED etc). * - * @param tasks Persistent tasks + * @param tasks Persistent tasks. If null an empty set is returned. * @return The job Ids of anomaly detector job tasks */ - public static Set openJobIds(PersistentTasksCustomMetaData tasks) { + public static Set openJobIds(@Nullable PersistentTasksCustomMetaData tasks) { + if (tasks == null) { + return Collections.emptySet(); + } + return tasks.findTasks(JOB_TASK_NAME, task -> true) .stream() .map(t -> t.getId().substring(JOB_TASK_ID_PREFIX.length())) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index e2db7c3a30951..c3579fe4173b8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -31,6 +31,10 @@ public void testGetJobState() { assertEquals(JobState.OPENED, MlTasks.getJobState("foo", tasksBuilder.build())); } + public void testGetJobState_GivenNull() { + assertEquals(JobState.CLOSED, MlTasks.getJobState("foo", null)); + } + public void testGetDatefeedState() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); // A missing task is a stopped datafeed @@ -83,6 +87,10 @@ public void testOpenJobIds() { assertThat(MlTasks.openJobIds(tasksBuilder.build()), containsInAnyOrder("foo-1", "bar")); } + public void testOpenJobIds_GivenNull() { + assertThat(MlTasks.openJobIds(null), empty()); + } + public void testTaskExistsForJob() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); assertFalse(MlTasks.taskExistsForJob("job-1", tasksBuilder.build())); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java index 96ba9e6fbbebf..e982e3886f6a6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java @@ -8,40 +8,37 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction; import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder; import java.util.Collections; -import java.util.List; import java.util.function.Supplier; public class TransportGetCalendarEventsAction extends HandledTransportAction { private final JobResultsProvider jobResultsProvider; - private final ClusterService clusterService; + private final JobConfigProvider jobConfigProvider; @Inject public TransportGetCalendarEventsAction(Settings settings, TransportService transportService, - ActionFilters actionFilters, ClusterService clusterService, - JobResultsProvider jobResultsProvider) { + ActionFilters actionFilters, JobResultsProvider jobResultsProvider, + JobConfigProvider jobConfigProvider) { super(settings, GetCalendarEventsAction.NAME, transportService, actionFilters, (Supplier) GetCalendarEventsAction.Request::new); this.jobResultsProvider = jobResultsProvider; - this.clusterService = clusterService; + this.jobConfigProvider = jobConfigProvider; } @Override @@ -67,26 +64,28 @@ protected void doExecute(Task task, GetCalendarEventsAction.Request request, ); if (request.getJobId() != null) { - ClusterState state = clusterService.state(); - MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(state); - List jobGroups; - String requestId = request.getJobId(); + jobConfigProvider.getJob(request.getJobId(), ActionListener.wrap( + jobBuiler -> { + Job job = jobBuiler.build(); + jobResultsProvider.scheduledEventsForJob(request.getJobId(), job.getGroups(), query, eventsListener); - Job job = currentMlMetadata.getJobs().get(request.getJobId()); - if (job == null) { - // Check if the requested id is a job group - if (currentMlMetadata.isGroupOrJob(request.getJobId()) == false) { - listener.onFailure(ExceptionsHelper.missingJobException(request.getJobId())); - return; - } - jobGroups = Collections.singletonList(request.getJobId()); - requestId = null; - } else { - jobGroups = job.getGroups(); - } - - jobResultsProvider.scheduledEventsForJob(requestId, jobGroups, query, eventsListener); + }, + jobNotFound -> { + // is the request Id a group? + jobConfigProvider.groupExists(request.getJobId(), ActionListener.wrap( + groupExists -> { + if (groupExists) { + jobResultsProvider.scheduledEventsForJob( + null, Collections.singletonList(request.getJobId()), query, eventsListener); + } else { + listener.onFailure(ExceptionsHelper.missingJobException(request.getJobId())); + } + }, + listener::onFailure + )); + } + )); } else { jobResultsProvider.scheduledEvents(query, eventsListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 8ca0c5ff531a0..55bd085bda27c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -630,6 +630,41 @@ public void expandGroupIds(List groupIds, ActionListener> l , client::search); } + /** + * Check if a group exists, that is there exists a job that is a member of + * the group. If there are one or more jobs that define the group then + * the listener responds with true else false. + * + * @param groupId The group Id + * @param listener Returns true, false or a failure + */ + public void groupExists(String groupId, ActionListener listener) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new TermQueryBuilder(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE)); + boolQueryBuilder.filter(new TermQueryBuilder(Job.GROUPS.getPreferredName(), groupId)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(boolQueryBuilder); + sourceBuilder.fetchSource(false); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setSize(0) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + listener.onResponse(response.getHits().totalHits > 0); + }, + listener::onFailure) + , client::search); + } + + /** + * Find jobs with custom rules defined. + * @param listener Jobs listener + */ public void findJobsWithCustomRules(ActionListener> listener) { String customRulesPath = Strings.collectionToDelimitedString(Arrays.asList(Job.ANALYSIS_CONFIG.getPreferredName(), AnalysisConfig.DETECTORS.getPreferredName(), Detector.CUSTOM_RULES_FIELD.getPreferredName()), "."); diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index a8d97f44b8d7d..bbb93ebd94322 100644 --- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -86,7 +86,8 @@ private void waitForTemplates() throws Exception { List templates = new ArrayList<>(); templates.addAll(Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, AnomalyDetectorsIndex.jobStateIndexName(), - AnomalyDetectorsIndex.jobResultsIndexPrefix())); + AnomalyDetectorsIndex.jobResultsIndexPrefix(), + AnomalyDetectorsIndex.configIndexName())); for (String template : templates) { awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(), From cb7b0e7a423d25553b965deb5f2be657c0a2c144 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 19 Oct 2018 15:49:58 +0100 Subject: [PATCH 16/57] [ML] Job in index: delete filter action (#34642) This changes the delete filter action to search for jobs using the filter to be deleted in the index rather than the cluster state. --- .../action/TransportDeleteFilterAction.java | 71 +++++++++++-------- .../ml/job/persistence/JobConfigProvider.java | 1 + 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java index 54acdf3712c7c..72d01643c2f6e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java @@ -16,24 +16,21 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlMetaIndex; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -42,25 +39,39 @@ public class TransportDeleteFilterAction extends HandledTransportAction { private final Client client; - private final ClusterService clusterService; + private final JobConfigProvider jobConfigProvider; @Inject public TransportDeleteFilterAction(Settings settings, TransportService transportService, - ActionFilters actionFilters, ClusterService clusterService, Client client) { + ActionFilters actionFilters, Client client, + JobConfigProvider jobConfigProvider) { super(settings, DeleteFilterAction.NAME, transportService, actionFilters, (Supplier) DeleteFilterAction.Request::new); - this.clusterService = clusterService; this.client = client; + this.jobConfigProvider = jobConfigProvider; } @Override protected void doExecute(Task task, DeleteFilterAction.Request request, ActionListener listener) { - final String filterId = request.getFilterId(); - ClusterState state = clusterService.state(); - Map jobs = MlMetadata.getMlMetadata(state).getJobs(); + jobConfigProvider.findJobsWithCustomRules(ActionListener.wrap( + jobs-> { + List currentlyUsedBy = findJobsUsingFilter(jobs, filterId); + if (!currentlyUsedBy.isEmpty()) { + listener.onFailure(ExceptionsHelper.conflictStatusException( + "Cannot delete filter, currently used by jobs: " + currentlyUsedBy)); + } else { + deleteFilter(filterId, listener); + } + }, + listener::onFailure + ) + ); + } + + private static List findJobsUsingFilter(List jobs, String filterId) { List currentlyUsedBy = new ArrayList<>(); - for (Job job : jobs.values()) { + for (Job job : jobs) { List detectors = job.getAnalysisConfig().getDetectors(); for (Detector detector : detectors) { if (detector.extractReferencedFilters().contains(filterId)) { @@ -69,31 +80,31 @@ protected void doExecute(Task task, DeleteFilterAction.Request request, ActionLi } } } - if (!currentlyUsedBy.isEmpty()) { - throw ExceptionsHelper.conflictStatusException("Cannot delete filter, currently used by jobs: " - + currentlyUsedBy); - } + return currentlyUsedBy; + } - DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId)); + private void deleteFilter(String filterId, ActionListener listener) { + DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, + MlFilter.documentId(filterId)); BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); bulkRequestBuilder.add(deleteRequest); bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); executeAsyncWithOrigin(client, ML_ORIGIN, BulkAction.INSTANCE, bulkRequestBuilder.request(), - new ActionListener() { - @Override - public void onResponse(BulkResponse bulkResponse) { - if (bulkResponse.getItems()[0].status() == RestStatus.NOT_FOUND) { - listener.onFailure(new ResourceNotFoundException("Could not delete filter with ID [" + filterId - + "] because it does not exist")); - } else { - listener.onResponse(new AcknowledgedResponse(true)); - } + new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + if (bulkResponse.getItems()[0].status() == RestStatus.NOT_FOUND) { + listener.onFailure(new ResourceNotFoundException("Could not delete filter with ID [" + filterId + + "] because it does not exist")); + } else { + listener.onResponse(new AcknowledgedResponse(true)); } + } - @Override - public void onFailure(Exception e) { - listener.onFailure(ExceptionsHelper.serverError("Could not delete filter with ID [" + filterId + "]", e)); - } - }); + @Override + public void onFailure(Exception e) { + listener.onFailure(ExceptionsHelper.serverError("Could not delete filter with ID [" + filterId + "]", e)); + } + }); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 55bd085bda27c..3007f9241221c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -107,6 +107,7 @@ public void putJob(Job job, ActionListener listener) { ElasticsearchMappings.DOC_TYPE, Job.documentId(job.getId())) .setSource(source) .setOpType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .request(); executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( From 2194128b708b1737999a2818a9bd313c28535194 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 29 Oct 2018 16:45:54 +0000 Subject: [PATCH 17/57] [ML] Job in Index: Enable integ tests (#34851) Enables the ml integration tests excluding the rolling upgrade tests and a lot of fixes to make the tests pass again. --- .../xpack/core/ml/action/OpenJobAction.java | 5 ++ .../xpack/core/ml/job/config/Job.java | 5 ++ .../persistence/ElasticsearchMappings.java | 23 ++------ .../ml/job/results/ReservedFieldNames.java | 7 +-- .../xpack/core/ml/action/JobParamsTests.java | 13 ++++ .../xpack/core/ml/job/config/JobTests.java | 8 +++ .../ml/integration/DatafeedJobsRestIT.java | 2 +- .../ml/integration/DeleteExpiredDataIT.java | 4 +- .../ml/action/TransportDeleteJobAction.java | 2 +- .../ml/action/TransportGetBucketsAction.java | 4 +- .../action/TransportGetCategoriesAction.java | 4 +- .../action/TransportGetDatafeedsAction.java | 15 +++-- .../action/TransportGetInfluencersAction.java | 4 +- .../TransportGetModelSnapshotsAction.java | 4 +- .../TransportGetOverallBucketsAction.java | 3 +- .../ml/action/TransportGetRecordsAction.java | 4 +- .../ml/action/TransportOpenJobAction.java | 12 ---- .../ml/action/TransportPutDatafeedAction.java | 2 - .../persistence/DatafeedConfigProvider.java | 20 +++++-- .../xpack/ml/job/JobManager.java | 51 +++++++++++++--- .../ml/job/persistence/JobConfigProvider.java | 59 ++++++++++++++++--- .../output/AutoDetectResultProcessor.java | 49 ++++++++++++--- .../MachineLearningLicensingTests.java | 11 +--- .../action/TransportCloseJobActionTests.java | 12 ++-- .../action/TransportOpenJobActionTests.java | 14 +++-- .../integration/BasicDistributedJobsIT.java | 38 ++++++++++-- .../integration/DatafeedConfigProviderIT.java | 8 ++- .../ml/integration/JobConfigProviderIT.java | 23 ++++---- .../xpack/ml/integration/TooManyJobsIT.java | 2 + .../xpack/ml/job/JobManagerTests.java | 8 +-- .../AutoDetectResultProcessorTests.java | 23 +++++++- .../rest-api-spec/test/ml/forecast.yml | 4 ++ .../rest-api-spec/test/ml/jobs_crud.yml | 13 ++-- .../rest-api-spec/test/ml/post_data.yml | 7 +++ x-pack/qa/rolling-upgrade/build.gradle | 34 ++++++++++- 35 files changed, 355 insertions(+), 142 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index 738eeb7f28335..69414c759d7aa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -135,6 +135,7 @@ public static class JobParams implements XPackPlugin.XPackPersistentTaskParams { /** TODO Remove in 7.0.0 */ public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); public static final ParseField TIMEOUT = new ParseField("timeout"); + public static final ParseField JOB = new ParseField("job"); public static ObjectParser PARSER = new ObjectParser<>(MlTasks.JOB_TASK_NAME, true, JobParams::new); static { @@ -142,6 +143,7 @@ public static class JobParams implements XPackPlugin.XPackPersistentTaskParams { PARSER.declareBoolean((p, v) -> {}, IGNORE_DOWNTIME); PARSER.declareString((params, val) -> params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); + PARSER.declareObject(JobParams::setJob, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOB); } public static JobParams fromXContent(XContentParser parser) { @@ -221,6 +223,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.startObject(); builder.field(Job.ID.getPreferredName(), jobId); builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep()); + if (job != null) { + builder.field("job", job); + } builder.endObject(); // The job field is streamed but not persisted return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index de44ee6f7ee5e..5b244ba44d5c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Nullable; @@ -1094,6 +1095,10 @@ private void validateGroups() { if (MlStrings.isValidId(group) == false) { throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_GROUP, group)); } + if (this.id.equals(group)) { + // cannot have a group name the same as the job id + throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, group)); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 55dc288b7cfa4..491be55049c10 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -11,7 +11,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; @@ -156,21 +155,7 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio .field(TYPE, KEYWORD) .endObject() .startObject(AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName()) - .startObject(PROPERTIES) - .startObject(CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - // TOKENIZER, TOKEN_FILTERS and CHAR_FILTERS are complex types, don't parse or index - .startObject(CategorizationAnalyzerConfig.TOKENIZER.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName()) - .field(ENABLED, false) - .endObject() - .endObject() + .field(ENABLED, false) .endObject() .startObject(AnalysisConfig.LATENCY.getPreferredName()) .field(TYPE, KEYWORD) @@ -297,9 +282,6 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio .startObject(Job.FINISHED_TIME.getPreferredName()) .field(TYPE, DATE) .endObject() - .startObject(Job.LAST_DATA_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() .startObject(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName()) .field(TYPE, LONG) // TODO should be ByteSizeValue .endObject() @@ -330,6 +312,9 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio .startObject(Job.MODEL_SNAPSHOT_ID.getPreferredName()) .field(TYPE, KEYWORD) .endObject() + .startObject(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() .startObject(Job.RESULTS_INDEX_NAME.getPreferredName()) .field(TYPE, KEYWORD) .endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index baff9f3a2d51d..400fcc07f4c53 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -9,7 +9,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; @@ -189,7 +188,6 @@ public final class ReservedFieldNames { Job.DATA_DESCRIPTION.getPreferredName(), Job.DESCRIPTION.getPreferredName(), Job.FINISHED_TIME.getPreferredName(), - Job.LAST_DATA_TIME.getPreferredName(), Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), Job.MODEL_PLOT_CONFIG.getPreferredName(), Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(), @@ -197,6 +195,7 @@ public final class ReservedFieldNames { Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), Job.RESULTS_RETENTION_DAYS.getPreferredName(), Job.MODEL_SNAPSHOT_ID.getPreferredName(), + Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), Job.RESULTS_INDEX_NAME.getPreferredName(), AnalysisConfig.BUCKET_SPAN.getPreferredName(), @@ -214,10 +213,6 @@ public final class ReservedFieldNames { AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(), AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), - CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName(), - CategorizationAnalyzerConfig.TOKENIZER.getPreferredName(), - CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName(), - Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName(), Detector.FUNCTION_FIELD.getPreferredName(), Detector.FIELD_NAME_FIELD.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java index 740b01abf0dda..03fb553e61e07 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java @@ -10,8 +10,10 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; import java.io.IOException; +import java.util.function.Predicate; public class JobParamsTests extends AbstractSerializingTestCase { @@ -25,6 +27,9 @@ public static OpenJobAction.JobParams createJobParams() { if (randomBoolean()) { params.setTimeout(TimeValue.timeValueMillis(randomNonNegativeLong())); } + if (randomBoolean()) { + params.setJob(JobTests.createRandomizedJob()); + } return params; } @@ -42,4 +47,12 @@ protected Writeable.Reader instanceReader() { protected boolean supportsUnknownFields() { return true; } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // Don't insert random fields into the job object as the + // custom_fields member accepts arbitrary fields and new + // fields inserted there will result in object inequality + return path -> path.startsWith(OpenJobAction.JobParams.JOB.getPreferredName()); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 4fa6617f045f6..55b84995d58f7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -7,6 +7,7 @@ import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; @@ -506,6 +507,13 @@ public void testInvalidGroup() { assertThat(e.getMessage(), containsString("Invalid group id '$$$'")); } + public void testInvalidGroup_matchesJobId() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setGroups(Collections.singletonList("foo")); + ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, builder::build); + assertEquals(e.getMessage(), "job and group names must be unique but job [foo] and group [foo] have the same name"); + } + public void testEstimateMemoryFootprint_GivenEstablished() { Job.Builder builder = buildJobBuilder("established"); long establishedModelMemory = randomIntBetween(10_000, 2_000_000_000); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 5772d0be42853..ce4320a26a719 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -704,7 +704,7 @@ public void testRealtime() throws Exception { response = e.getResponse(); assertThat(response.getStatusLine().getStatusCode(), equalTo(409)); assertThat(EntityUtils.toString(response.getEntity()), - containsString("Cannot delete job [" + jobId + "] because datafeed [" + datafeedId + "] refers to it")); + containsString("Cannot delete job [" + jobId + "] because the job is opened")); response = client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stop")); assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index e5aaf5f4fdb10..0f860043b67b0 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -95,8 +95,8 @@ public void testDeleteExpiredDataGivenNothingToDelete() throws Exception { } public void testDeleteExpiredData() throws Exception { - registerJob(newJobBuilder("no-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(null)); - registerJob(newJobBuilder("results-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(null)); + registerJob(newJobBuilder("no-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(1000L)); + registerJob(newJobBuilder("results-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(1000L)); registerJob(newJobBuilder("snapshots-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(2L)); registerJob(newJobBuilder("snapshots-retention-with-retain").setResultsRetentionDays(null).setModelSnapshotRetentionDays(2L)); registerJob(newJobBuilder("results-and-snapshots-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(2L)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index d8e8b8f8178f7..761c21b63f165 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -331,7 +331,7 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri builder -> { Job job = builder.build(); indexName.set(job.getResultsIndexName()); - if (indexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + + if (indexName.get().equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT)) { //don't bother searching the index any further, we are on the default shared customIndexSearchHandler.onResponse(null); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java index 5e0bafac71475..65a2f17847786 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java @@ -39,8 +39,8 @@ public TransportGetBucketsAction(Settings settings, TransportService transportSe @Override protected void doExecute(Task task, GetBucketsAction.Request request, ActionListener listener) { - jobManager.getJob(request.getJobId(), ActionListener.wrap( - job -> { + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + ok -> { BucketsQueryBuilder query = new BucketsQueryBuilder().expand(request.isExpand()) .includeInterim(request.isExcludeInterim() == false) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java index d90cd59ed0215..9f6a651d9b47a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java @@ -38,8 +38,8 @@ public TransportGetCategoriesAction(Settings settings, TransportService transpor @Override protected void doExecute(Task task, GetCategoriesAction.Request request, ActionListener listener) { - jobManager.getJob(request.getJobId(), ActionListener.wrap( - job -> { + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + jobExists -> { Integer from = request.getPageParams() != null ? request.getPageParams().getFrom() : null; Integer size = request.getPageParams() != null ? request.getPageParams().getSize() : null; jobResultsProvider.categoryDefinitions(request.getJobId(), request.getCategoryId(), true, from, size, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java index 1acf095102bdc..aa5b174a92a05 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java @@ -97,12 +97,15 @@ Map expandClusterStateDatafeeds(String datafeedExpressio ClusterState clusterState) { Map configById = new HashMap<>(); - - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds); - - for (String expandedDatafeedId : expandedDatafeedIds) { - configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId)); + try { + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + Set expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds); + + for (String expandedDatafeedId : expandedDatafeedIds) { + configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId)); + } + } catch (Exception e){ + // ignore } return configById; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java index 9967262ba1070..5004e936ce646 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java @@ -39,8 +39,8 @@ public TransportGetInfluencersAction(Settings settings, TransportService transpo @Override protected void doExecute(Task task, GetInfluencersAction.Request request, ActionListener listener) { - jobManager.getJob(request.getJobId(), ActionListener.wrap( - job -> { + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + jobExists -> { InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder() .includeInterim(request.isExcludeInterim() == false) .start(request.getStart()) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java index a690d6ef8593c..ab340e47851b6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java @@ -43,8 +43,8 @@ protected void doExecute(Task task, GetModelSnapshotsAction.Request request, request.getJobId(), request.getSnapshotId(), request.getPageParams().getFrom(), request.getPageParams().getSize(), request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder()); - jobManager.getJob(request.getJobId(), ActionListener.wrap( - job -> { + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + ok -> { jobResultsProvider.modelSnapshots(request.getJobId(), request.getPageParams().getFrom(), request.getPageParams().getSize(), request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), request.getSnapshotId(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java index 3364f8e08b85b..48de690946b00 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java @@ -75,7 +75,8 @@ public TransportGetOverallBucketsAction(Settings settings, ThreadPool threadPool } @Override - protected void doExecute(Task task, GetOverallBucketsAction.Request request, ActionListener listener) { + protected void doExecute(Task task, GetOverallBucketsAction.Request request, + ActionListener listener) { jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap( jobPage -> { if (jobPage.count() == 0) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java index 6c43985af5561..8957ecce9bd69 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java @@ -40,8 +40,8 @@ public TransportGetRecordsAction(Settings settings, TransportService transportSe @Override protected void doExecute(Task task, GetRecordsAction.Request request, ActionListener listener) { - jobManager.getJob(request.getJobId(), ActionListener.wrap( - job -> { + jobManager.jobExists(request.getJobId(), ActionListener.wrap( + jobExists -> { RecordsQueryBuilder query = new RecordsQueryBuilder() .includeInterim(request.isExcludeInterim() == false) .epochStart(request.getStart()) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index daa1c789f12ab..e116c75842486 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -170,14 +170,6 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j continue; } - if (nodeSupportsMlJobs(node.getVersion()) == false) { - String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) - + "], because this node does not support machine learning jobs"; - logger.trace(reason); - reasons.add(reason); - continue; - } - if (nodeSupportsModelSnapshotVersion(node, job) == false) { String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because the job's model snapshot requires a node of version [" @@ -351,10 +343,6 @@ private static boolean nodeSupportsModelSnapshotVersion(DiscoveryNode node, Job return node.getVersion().onOrAfter(job.getModelSnapshotMinVersion()); } - private static boolean nodeSupportsMlJobs(Version nodeVersion) { - return nodeVersion.onOrAfter(Version.V_5_5_0); - } - private static boolean jobHasRules(Job job) { return job.getAnalysisConfig().getDetectors().stream().anyMatch(d -> d.getRules().isEmpty() == false); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java index 874ee8f71f5ad..b2b12f40d0691 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java @@ -51,7 +51,6 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction headers ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)) .setSource(source) .setOpType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .request(); executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( @@ -181,19 +186,20 @@ public void onFailure(Exception e) { public void findDatafeedsForJobIds(Collection jobIds, ActionListener> listener) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdsQuery(jobIds)); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName()); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSize(jobIds.size()) .setSource(sourceBuilder).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( response -> { Set datafeedIds = new HashSet<>(); - SearchHit[] hits = response.getHits().getHits(); // There cannot be more than one datafeed per job - assert hits.length <= jobIds.size(); + assert response.getHits().totalHits <= jobIds.size(); + SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { datafeedIds.add(hit.field(DatafeedConfig.ID.getPreferredName()).getValue()); @@ -214,6 +220,7 @@ public void findDatafeedsForJobIds(Collection jobIds, ActionListener actionListener) { DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId)); + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener() { @Override public void onResponse(DeleteResponse deleteResponse) { @@ -307,6 +314,7 @@ private void indexUpdatedConfig(DatafeedConfig updatedConfig, long version, Acti ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(updatedConfig.getId())) .setSource(updatedSource) .setVersion(version) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .request(); executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); @@ -341,12 +349,12 @@ private void indexUpdatedConfig(DatafeedConfig updatedConfig, long version, Acti * wildcard then setting this true will not suppress the exception * @param listener The expanded datafeed IDs listener */ - public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, ActionListener> listener) { + public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName()); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -357,7 +365,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( response -> { - Set datafeedIds = new HashSet<>(); + SortedSet datafeedIds = new TreeSet<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { datafeedIds.add(hit.field(DatafeedConfig.ID.getPreferredName()).getValue()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 80612d6b51402..83f3ee2a3b78c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.job; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexResponse; @@ -198,11 +199,15 @@ public void expandJobs(String expression, boolean allowNoJobs, ActionListener expandJobsFromClusterState(String expression, boolean allowNoJobs, ClusterState clusterState) { - Set expandedJobIds = MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoJobs); - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); Map jobIdToJob = new HashMap<>(); - for (String expandedJobId : expandedJobIds) { - jobIdToJob.put(expandedJobId, mlMetadata.getJobs().get(expandedJobId)); + try { + Set expandedJobIds = MlMetadata.getMlMetadata(clusterState).expandJobIds(expression, allowNoJobs); + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + for (String expandedJobId : expandedJobIds) { + jobIdToJob.put(expandedJobId, mlMetadata.getJobs().get(expandedJobId)); + } + } catch (Exception e) { + // ignore } return jobIdToJob; } @@ -274,9 +279,39 @@ public void onFailure(Exception e) { } }; - ActionListener checkForLeftOverDocs = ActionListener.wrap( - response -> { - jobResultsProvider.createJobResultIndex(job, state, putJobListener); + ActionListener> checkForLeftOverDocs = ActionListener.wrap( + matchedIds -> { + if (matchedIds.isEmpty()) { + jobResultsProvider.createJobResultIndex(job, state, putJobListener); + } else { + // A job has the same Id as one of the group names + // error with the first in the list + actionListener.onFailure(new ResourceAlreadyExistsException( + Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, matchedIds.get(0)))); + } + }, + actionListener::onFailure + ); + + ActionListener checkNoJobsWithGroupId = ActionListener.wrap( + groupExists -> { + if (groupExists) { + actionListener.onFailure(new ResourceAlreadyExistsException( + Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, job.getId()))); + return; + } + if (job.getGroups().isEmpty()) { + checkForLeftOverDocs.onResponse(Collections.emptyList()); + } else { + jobConfigProvider.jobIdMatches(job.getGroups(), checkForLeftOverDocs); + } + }, + actionListener::onFailure + ); + + ActionListener checkNoGroupWithTheJobId = ActionListener.wrap( + ok -> { + jobConfigProvider.groupExists(job.getId(), checkNoJobsWithGroupId); }, actionListener::onFailure ); @@ -286,7 +321,7 @@ public void onFailure(Exception e) { if (jobExists) { actionListener.onFailure(ExceptionsHelper.jobAlreadyExists(job.getId())); } else { - jobResultsProvider.checkForLeftOverDocuments(job, checkForLeftOverDocs); + jobResultsProvider.checkForLeftOverDocuments(job, checkNoGroupWithTheJobId); } }, actionListener::onFailure diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 3007f9241221c..64d825a09347b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -55,7 +55,9 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -74,6 +76,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -223,6 +227,7 @@ public void onFailure(Exception e) { public void deleteJob(String jobId, boolean errorIfMissing, ActionListener actionListener) { DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener() { @Override @@ -372,6 +377,7 @@ private void indexUpdatedJob(Job updatedJob, long version, ActionListener u ElasticsearchMappings.DOC_TYPE, Job.documentId(updatedJob.getId())) .setSource(updatedSource) .setVersion(version) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .request(); executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( @@ -436,6 +442,41 @@ public void onFailure(Exception e) { }); } + /** + * For the list of job Ids find all that match existing jobs Ids. + * The repsonse is all the job Ids in {@code ids} that match an existing + * job Id. + * @param ids Job Ids to find + * @param listener The matched Ids listener + */ + public void jobIdMatches(List ids, ActionListener> listener) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new TermQueryBuilder(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE)); + boolQueryBuilder.filter(new TermsQueryBuilder(Job.ID.getPreferredName(), ids)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + sourceBuilder.fetchSource(false); + sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + + SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSize(ids.size()) + .setSource(sourceBuilder).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + SearchHit[] hits = response.getHits().getHits(); + List matchedIds = new ArrayList<>(); + for (SearchHit hit : hits) { + matchedIds.add(hit.field(Job.ID.getPreferredName()).getValue()); + } + listener.onResponse(matchedIds); + }, + listener::onFailure) + , client::search); + } + /** * Sets the job's {@code deleting} field to true * @param jobId The job to mark as deleting @@ -494,13 +535,13 @@ public void markJobAsDeleting(String jobId, ActionListener listener) { * @param excludeDeleting If true exclude jobs marked as deleting * @param listener The expanded job Ids listener */ - public void expandJobsIds(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener> listener) { + public void expandJobsIds(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener> listener) { String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName()); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName()); + sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -511,8 +552,8 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( response -> { - Set jobIds = new HashSet<>(); - Set groupsIds = new HashSet<>(); + SortedSet jobIds = new TreeSet<>(); + SortedSet groupsIds = new TreeSet<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { jobIds.add(hit.field(Job.ID.getPreferredName()).getValue()); @@ -605,12 +646,12 @@ public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDe * @param groupIds Group Ids to expand * @param listener Expanded job Ids listener */ - public void expandGroupIds(List groupIds, ActionListener> listener) { + public void expandGroupIds(List groupIds, ActionListener> listener) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(new TermsQueryBuilder(Job.GROUPS.getPreferredName(), groupIds)); - sourceBuilder.sort(Job.ID.getPreferredName()); + sourceBuilder.sort(Job.ID.getPreferredName(), SortOrder.DESC); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName()); + sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -619,7 +660,7 @@ public void expandGroupIds(List groupIds, ActionListener> l executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( response -> { - Set jobIds = new HashSet<>(); + SortedSet jobIds = new TreeSet<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { jobIds.add(hit.field(Job.ID.getPreferredName()).getValue()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java index e07e6d6bc7e6d..b4da9b02aeb2f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; @@ -101,6 +102,7 @@ public class AutoDetectResultProcessor { final CountDownLatch completionLatch = new CountDownLatch(1); final Semaphore updateModelSnapshotSemaphore = new Semaphore(1); + volatile CountDownLatch onCloseActionsLatch; private final FlushListener flushListener; private volatile boolean processKilled; private volatile boolean failed; @@ -122,9 +124,9 @@ public AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, R restoredSnapshot, new FlushListener()); } - AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, JobResultsPersister persister, - JobResultsProvider jobResultsProvider, ModelSizeStats latestModelSizeStats, boolean restoredSnapshot, - FlushListener flushListener) { + AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, + JobResultsPersister persister, JobResultsProvider jobResultsProvider, ModelSizeStats latestModelSizeStats, + boolean restoredSnapshot, FlushListener flushListener) { this.client = Objects.requireNonNull(client); this.auditor = Objects.requireNonNull(auditor); this.jobId = Objects.requireNonNull(jobId); @@ -170,9 +172,18 @@ public void process(AutodetectProcess process) { } catch (Exception e) { LOGGER.warn(new ParameterizedMessage("[{}] Error persisting autodetect results", jobId), e); } + if (processKilled == false) { + try { + onAutodetectClose(); + } catch (Exception e) { + if (onCloseActionsLatch != null) { + onCloseActionsLatch.countDown(); + } + throw e; + } + } LOGGER.info("[{}] {} buckets parsed from autodetect output", jobId, bucketCount); - onAutodetectClose(); } catch (Exception e) { failed = true; @@ -432,9 +443,22 @@ private synchronized void runEstablishedModelMemoryUpdate(boolean cancelExisting } private void onAutodetectClose() { - updateJob(jobId, Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date()), ActionListener.wrap( - r -> runEstablishedModelMemoryUpdate(true), - e -> LOGGER.error("[" + jobId + "] Failed to finalize job on autodetect close", e)) + onCloseActionsLatch = new CountDownLatch(1); + + ActionListener updateListener = ActionListener.wrap( + updateResponse -> { + runEstablishedModelMemoryUpdate(true); + onCloseActionsLatch.countDown(); + }, + e -> { + LOGGER.error("[" + jobId + "] Failed to finalize job on autodetect close", e); + onCloseActionsLatch.countDown(); + } + ); + + updateJob(jobId, Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date()), + new ThreadedActionListener<>(LOGGER, client.threadPool(), + MachineLearning.UTILITY_THREAD_POOL_NAME, updateListener, false) ); } @@ -467,11 +491,12 @@ public void onFailure(Exception e) { }, e -> LOGGER.error("[" + jobId + "] Failed to calculate established model memory", e)); } - private void updateJob(String jobId, Map update, ActionListener listener) { + private void updateJob(String jobId, Map update, ActionListener listener) { UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); updateRequest.retryOnConflict(3); updateRequest.doc(update); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); executeAsyncWithOrigin(client, ML_ORIGIN, UpdateAction.INSTANCE, updateRequest, listener); } @@ -483,6 +508,14 @@ public void awaitCompletion() throws TimeoutException { TimeUnit.MINUTES) == false) { throw new TimeoutException("Timed out waiting for results processor to complete for job " + jobId); } + + // Once completionLatch has passed then onCloseActionsLatch must either + // be set or null, it will not be set later. + if (onCloseActionsLatch != null && onCloseActionsLatch.await( + MachineLearningField.STATE_PERSIST_RESTORE_TIMEOUT.getMinutes(), TimeUnit.MINUTES) == false) { + throw new TimeoutException("Timed out waiting for results processor run post close actions " + jobId); + } + // Input stream has been completely processed at this point. // Wait for any updateModelSnapshotOnJob calls to complete. updateModelSnapshotSemaphore.acquire(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java index 29108f46c72c1..287bd22f91f92 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobState; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.LocalStateMachineLearning; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.junit.Before; @@ -88,7 +89,6 @@ public void testMachineLearningPutJobActionRestricted() { } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningOpenJobActionRestricted() throws Exception { String jobId = "testmachinelearningopenjobactionrestricted"; assertMLAllowed(true); @@ -140,7 +140,6 @@ public void testMachineLearningOpenJobActionRestricted() throws Exception { } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningPutDatafeedActionRestricted() throws Exception { String jobId = "testmachinelearningputdatafeedactionrestricted"; String datafeedId = jobId + "-datafeed"; @@ -188,7 +187,6 @@ public void testMachineLearningPutDatafeedActionRestricted() throws Exception { } } - @AwaitsFix(bugUrl = "JIndex development") public void testAutoCloseJobWithDatafeed() throws Exception { String jobId = "testautoclosejobwithdatafeed"; String datafeedId = jobId + "-datafeed"; @@ -229,6 +227,8 @@ public void testAutoCloseJobWithDatafeed() throws Exception { } assertMLAllowed(false); + client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); + // now that the license is invalid, the job should be closed and datafeed stopped: assertBusy(() -> { JobState jobState = getJobStats(jobId).getState(); @@ -291,7 +291,6 @@ public void testAutoCloseJobWithDatafeed() throws Exception { }); } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningStartDatafeedActionRestricted() throws Exception { String jobId = "testmachinelearningstartdatafeedactionrestricted"; String datafeedId = jobId + "-datafeed"; @@ -366,7 +365,6 @@ public void testMachineLearningStartDatafeedActionRestricted() throws Exception } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningStopDatafeedActionNotRestricted() throws Exception { String jobId = "testmachinelearningstopdatafeedactionnotrestricted"; String datafeedId = jobId + "-datafeed"; @@ -433,7 +431,6 @@ public void testMachineLearningStopDatafeedActionNotRestricted() throws Exceptio } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningCloseJobActionNotRestricted() throws Exception { String jobId = "testmachinelearningclosejobactionnotrestricted"; assertMLAllowed(true); @@ -477,7 +474,6 @@ public void testMachineLearningCloseJobActionNotRestricted() throws Exception { } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningDeleteJobActionNotRestricted() throws Exception { String jobId = "testmachinelearningclosejobactionnotrestricted"; assertMLAllowed(true); @@ -503,7 +499,6 @@ public void testMachineLearningDeleteJobActionNotRestricted() throws Exception { } } - @AwaitsFix(bugUrl = "JIndex development") public void testMachineLearningDeleteDatafeedActionNotRestricted() throws Exception { String jobId = "testmachinelearningdeletedatafeedactionnotrestricted"; String datafeedId = jobId + "-datafeed"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 2348bc1849932..a2c546930f22b 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -42,6 +41,8 @@ import java.util.Date; import java.util.List; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -218,8 +219,10 @@ public void testDoExecute_whenNothingToClose() { TransportCloseJobAction transportAction = createAction(); when(clusterService.state()).thenReturn(clusterState); - mockJobConfigProviderExpandIds(Collections.singleton("foo")); - mockDatafeedConfigFindDatafeeds(Collections.emptySet()); + SortedSet expandedIds = new TreeSet<>(); + expandedIds.add("foo"); + mockJobConfigProviderExpandIds(expandedIds); + mockDatafeedConfigFindDatafeeds(Collections.emptySortedSet()); AtomicBoolean gotResponse = new AtomicBoolean(false); CloseJobAction.Request request = new Request("foo"); @@ -235,7 +238,8 @@ public void onResponse(CloseJobAction.Response response) { @Override public void onFailure(Exception e) { - fail(); + assertNull(e.getMessage(), e); + } }); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index d3056680231e3..32ae117f9fd45 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -353,14 +353,18 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); MetaData.Builder metaData = MetaData.builder(); + + Job job = BaseMlIntegTestCase.createFareQuoteJob("job_with_incompatible_model_snapshot") + .setModelSnapshotId("incompatible_snapshot") + .setModelSnapshotMinVersion(Version.V_6_3_0) + .build(new Date()); cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - - Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); - - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, logger); - assertThat(result.getExplanation(), containsString("because this node does not support machine learning jobs")); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_incompatible_model_snapshot", job, cs.build(), + 2, 10, 30, logger); + assertThat(result.getExplanation(), containsString( + "because the job's model snapshot requires a node of version [6.3.0] or higher")); assertNull(result.getExecutorNode()); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 1303e2c747773..ce210bd07f85f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -323,20 +323,48 @@ public void testMaxConcurrentJobAllocations() throws Exception { assertEquals("Expected no violations, but got [" + violations + "]", 0, violations.size()); } - public void testMlIndicesNotAvailable() throws Exception { + // This test is designed to check that a job will not open when the .ml-state + // or .ml-anomalies-shared indices are not available. To do this those indices + // must be allocated on a node which is later stopped while .ml-config is + // allocated on a second node which remains active. + public void testMlStateAndResultsIndicesNotAvailable() throws Exception { internalCluster().ensureAtMostNumDataNodes(0); - // start non ml node, but that will hold the indices + // start non ml node that will hold the state and results indices logger.info("Start non ml node:"); internalCluster().startNode(Settings.builder() .put("node.data", true) + .put("node.attr.ml-indices", "state-and-results") .put(MachineLearning.ML_ENABLED.getKey(), false)); ensureStableCluster(1); + // start an ml node for the config index logger.info("Starting ml node"); String mlNode = internalCluster().startNode(Settings.builder() - .put("node.data", false) + .put("node.data", true) + .put("node.attr.ml-indices", "config") .put(MachineLearning.ML_ENABLED.getKey(), true)); ensureStableCluster(2); + // Create the indices (using installed templates) and set the routing to specific nodes + // State and results go on the state-and-results node, config goes on the config node + client().admin().indices().prepareCreate(".ml-anomalies-shared") + .setSettings(Settings.builder() + .put("index.routing.allocation.include.ml-indices", "state-and-results") + .put("index.routing.allocation.exclude.ml-indices", "config") + .build()) + .get(); + client().admin().indices().prepareCreate(".ml-state") + .setSettings(Settings.builder() + .put("index.routing.allocation.include.ml-indices", "state-and-results") + .put("index.routing.allocation.exclude.ml-indices", "config") + .build()) + .get(); + client().admin().indices().prepareCreate(".ml-config") + .setSettings(Settings.builder() + .put("index.routing.allocation.exclude.ml-indices", "state-and-results") + .put("index.routing.allocation.include.ml-indices", "config") + .build()) + .get(); + String jobId = "ml-indices-not-available-job"; Job.Builder job = createFareQuoteJob(jobId); PutJobAction.Request putJobRequest = new PutJobAction.Request(job); @@ -360,8 +388,8 @@ public void testMlIndicesNotAvailable() throws Exception { PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertEquals(0, tasks.taskMap().size()); }); - logger.info("Stop data node"); - internalCluster().stopRandomNode(settings -> settings.getAsBoolean("node.data", true)); + logger.info("Stop non ml node"); + internalCluster().stopRandomNode(settings -> settings.getAsBoolean(MachineLearning.ML_ENABLED.getKey(), false) == false); ensureStableCluster(1); Exception e = expectThrows(ElasticsearchStatusException.class, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index 8acee83e0b0b6..09d8c7ed41888 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -203,7 +204,7 @@ public void testUpdateWithValidatorFunctionThatErrors() throws Exception { } public void testAllowNoDatafeeds() throws InterruptedException { - AtomicReference> datafeedIdsHolder = new AtomicReference<>(); + AtomicReference> datafeedIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("_all", false, actionListener), @@ -246,7 +247,8 @@ public void testExpandDatafeeds() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); // Test job IDs only - Set expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("foo*", true, actionListener)); + SortedSet expandedIds = + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("foo*", true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); expandedIds = blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedIds("*-1", true, actionListener)); @@ -309,7 +311,7 @@ public void testFindDatafeedsForJobIds() throws Exception { blockingCall(actionListener -> datafeedConfigProvider.findDatafeedsForJobIds(Arrays.asList("j3", "j1"), actionListener), datafeedIdsHolder, exceptionHolder); - assertThat(datafeedIdsHolder.get(), containsInAnyOrder("bar-1", "foo-1")); + assertThat(datafeedIdsHolder.get(), contains("bar-1", "foo-1")); } public void testHeadersAreOverwritten() throws Exception { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index 75198a3350dc5..6f31fad179ba7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -38,10 +38,12 @@ import java.util.Date; import java.util.List; import java.util.Set; +import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -270,7 +272,7 @@ public void testUpdateWithValidator() throws Exception { } public void testAllowNoJobs() throws InterruptedException { - AtomicReference> jobIdsHolder = new AtomicReference<>(); + AtomicReference> jobIdsHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", false, true, actionListener), @@ -312,7 +314,8 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); // Job Ids - Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("_all", true, false, actionListener)); + SortedSet expandedIds = blockingCall(actionListener -> + jobConfigProvider.expandJobsIds("_all", true, false, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("tom", "dick", "harry", "harry-jnr")), expandedIds); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*", true, true, actionListener)); @@ -325,7 +328,7 @@ public void testExpandJobs_GroupsAndJobIds() throws Exception { assertEquals(new TreeSet<>(Arrays.asList("harry", "harry-jnr", "tom")), expandedIds); AtomicReference exceptionHolder = new AtomicReference<>(); - AtomicReference> jobIdsHolder = new AtomicReference<>(); + AtomicReference> jobIdsHolder = new AtomicReference<>(); blockingCall(actionListener -> jobConfigProvider.expandJobsIds("tom,missing1,missing2", true, false, actionListener), jobIdsHolder, exceptionHolder); assertNull(jobIdsHolder.get()); @@ -373,7 +376,7 @@ public void testExpandJobs_WildCardExpansion() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); // Test job IDs only - Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); + SortedSet expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("*-1", true, true,actionListener)); @@ -415,7 +418,7 @@ public void testExpandJobIds_excludeDeleting() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); - Set expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); + SortedSet expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, true, actionListener)); assertEquals(new TreeSet<>(Arrays.asList("foo-1", "foo-2")), expandedIds); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandJobsIds("foo*", true, false, actionListener)); @@ -445,17 +448,17 @@ public void testExpandGroups() throws Exception { client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.configIndexName()).get(); - Set expandedIds = blockingCall(actionListener -> + SortedSet expandedIds = blockingCall(actionListener -> jobConfigProvider.expandGroupIds(Collections.singletonList("fruit"), actionListener)); - assertThat(expandedIds, containsInAnyOrder("apples", "pears", "tomato")); + assertThat(expandedIds, contains("apples", "pears", "tomato")); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandGroupIds(Collections.singletonList("veg"), actionListener)); - assertThat(expandedIds, containsInAnyOrder("broccoli", "potato", "tomato")); + assertThat(expandedIds, contains("broccoli", "potato", "tomato")); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandGroupIds(Arrays.asList("fruit", "veg"), actionListener)); - assertThat(expandedIds, containsInAnyOrder("apples", "pears", "broccoli", "potato", "tomato")); + assertThat(expandedIds, contains("apples", "broccoli", "pears", "potato", "tomato")); expandedIds = blockingCall(actionListener -> jobConfigProvider.expandGroupIds(Collections.singletonList("unknown-group"), actionListener)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index c4150d633a8f0..87aa3c5b926e3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -123,10 +123,12 @@ public void testLazyNodeValidation() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34084") public void testSingleNode() throws Exception { verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 100)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34084") public void testMultipleNodes() throws Exception { verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 100)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index c593444297040..e7ec8c789855f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -67,7 +67,6 @@ import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests.addJobTask; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; @@ -296,7 +295,6 @@ public void testNotifyFilterChangedGivenNoop() { Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") public void testNotifyFilterChanged() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); @@ -316,10 +314,10 @@ public void testNotifyFilterChanged() throws IOException { docsAsBytes.add(toBytesReference(jobReferencingFilter2.build())); Job.Builder jobReferencingFilter3 = buildJobBuilder("job-referencing-filter-3"); - jobReferencingFilter2.setAnalysisConfig(filterAnalysisConfig); + jobReferencingFilter3.setAnalysisConfig(filterAnalysisConfig); + docsAsBytes.add(toBytesReference(jobReferencingFilter3.build())); Job.Builder jobWithoutFilter = buildJobBuilder("job-without-filter"); - docsAsBytes.add(toBytesReference(jobWithoutFilter.build())); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(jobReferencingFilter1.getId(), "node_id", JobState.OPENED, tasksBuilder); @@ -369,7 +367,6 @@ public void testNotifyFilterChanged() throws IOException { Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") public void testNotifyFilterChangedGivenOnlyAddedItems() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); @@ -406,7 +403,6 @@ public void testNotifyFilterChangedGivenOnlyAddedItems() throws IOException { Mockito.verifyNoMoreInteractions(auditor, updateJobProcessNotifier); } - @AwaitsFix(bugUrl = "Closed jobs are not audited when the filter changes") public void testNotifyFilterChangedGivenOnlyRemovedItems() throws IOException { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java index 37bd367c28e2c..05f83f4ae49cc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java @@ -8,10 +8,12 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -32,6 +34,7 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; @@ -49,6 +52,7 @@ import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -61,6 +65,7 @@ import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.same; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -89,10 +94,21 @@ public class AutoDetectResultProcessorTests extends ESTestCase { public void setUpMocks() { executor = new ScheduledThreadPoolExecutor(1); client = mock(Client.class); - auditor = mock(Auditor.class); + doAnswer(invocation -> { + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new UpdateResponse()); + return null; + }).when(client).execute(same(UpdateAction.INSTANCE), any(), any()); threadPool = mock(ThreadPool.class); when(client.threadPool()).thenReturn(threadPool); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + ExecutorService executorService = mock(ExecutorService.class); + org.elasticsearch.mock.orig.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); + auditor = mock(Auditor.class); renormalizer = mock(Renormalizer.class); persister = mock(JobResultsPersister.class); when(persister.persistModelSnapshot(any(), any())) @@ -121,7 +137,9 @@ public void testProcess() throws TimeoutException { processorUnderTest.process(process); processorUnderTest.awaitCompletion(); verify(renormalizer, times(1)).waitUntilIdle(); + verify(client, times(1)).execute(same(UpdateAction.INSTANCE), any(), any()); assertEquals(0, processorUnderTest.completionLatch.getCount()); + assertEquals(0, processorUnderTest.onCloseActionsLatch.getCount()); } public void testProcessResult_bucket() { @@ -480,6 +498,7 @@ public void testAwaitCompletion() throws TimeoutException { processorUnderTest.awaitCompletion(); assertEquals(0, processorUnderTest.completionLatch.getCount()); assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits()); + assertEquals(0, processorUnderTest.onCloseActionsLatch.getCount()); } public void testPersisterThrowingDoesntBlockProcessing() { @@ -533,6 +552,7 @@ public void testKill() throws TimeoutException { processorUnderTest.process(process); processorUnderTest.awaitCompletion(); + assertNull(processorUnderTest.onCloseActionsLatch); assertEquals(0, processorUnderTest.completionLatch.getCount()); assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits()); @@ -542,6 +562,7 @@ public void testKill() throws TimeoutException { verify(renormalizer).shutdown(); verify(renormalizer, times(1)).waitUntilIdle(); verify(flushListener, times(1)).clear(); + verify(client, never()).execute(same(UpdateAction.INSTANCE), any(), any()); } private void setupScheduleDelayTime(TimeValue delay) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml index df44751a37cd9..3677153d45b78 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml @@ -15,6 +15,10 @@ setup: --- "Test forecast unknown job": + - skip: + reason: "https://github.com/elastic/elasticsearch/issues/34747" + version: "6.5.0 - " + - do: catch: missing xpack.ml.forecast: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 3b08753e20913..140078df27218 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -1165,10 +1165,11 @@ - match: { job_id: "delimited-format-job" } --- -"Test job with named categorization_analyzer": +"Test jobs with named and custom categorization_analyzer": +# Check named and custom configs can share the same index & mappings - do: xpack.ml.put_job: - job_id: jobs-crud-categorization-analyzer-job + job_id: jobs-crud-named-categorization-analyzer-job body: > { "analysis_config" : { @@ -1180,14 +1181,12 @@ "data_description" : { } } - - match: { job_id: "jobs-crud-categorization-analyzer-job" } + - match: { job_id: "jobs-crud-named-categorization-analyzer-job" } - match: { analysis_config.categorization_analyzer: "standard" } ---- -"Test job with custom categorization_analyzer": - do: xpack.ml.put_job: - job_id: jobs-crud-categorization-analyzer-job + job_id: jobs-crud-custom-categorization-analyzer-job body: > { "analysis_config" : { @@ -1203,7 +1202,7 @@ "data_description" : { } } - - match: { job_id: "jobs-crud-categorization-analyzer-job" } + - match: { job_id: "jobs-crud-custom-categorization-analyzer-job" } - match: { analysis_config.categorization_analyzer.char_filter.0: "html_strip" } - match: { analysis_config.categorization_analyzer.tokenizer: "classic" } - match: { analysis_config.categorization_analyzer.filter.0: "stop" } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml index 7bc4f7df92acd..68590019234a7 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml @@ -187,6 +187,9 @@ setup: --- "Test POST data with invalid parameters": + - skip: + reason: "https://github.com/elastic/elasticsearch/issues/34747" + version: "6.5.0 - " - do: catch: missing @@ -234,6 +237,10 @@ setup: --- "Test Flush data with invalid parameters": + - skip: + reason: "https://github.com/elastic/elasticsearch/issues/34747" + version: "6.5.0 - " + - do: catch: missing xpack.ml.flush_job: diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 90da6cf4e58b8..cf4b17fa51236 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -162,6 +162,21 @@ subprojects { } setting 'xpack.watcher.encrypt_sensitive_data', 'true' } + + // Old versions of the code contain an invalid assertion that trips + // during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing + // the assertion, but this is impossible for released versions. + // However, released versions run without assertions, so end users won't + // be suffering the effects. This argument effectively removes the + // incorrect assertion from the older versions used in the BWC tests. + if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) { + jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk' + } + + systemProperty 'tests.rest.blacklist', [ + 'old_cluster/30_ml_jobs_crud/*', + 'old_cluster/40_ml_datafeed_crud/*', + ].join(',') } Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner") @@ -203,6 +218,11 @@ subprojects { if (version.before('6.0.0')) { keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase' } + + systemProperty 'tests.rest.blacklist', [ + 'mixed_cluster/30_ml_jobs_crud/*', + 'mixed_cluster/40_ml_datafeed_crud/*', + ].join(',') } } @@ -219,8 +239,8 @@ subprojects { // We only need to run these tests once so we may as well do it when we're two thirds upgraded systemProperty 'tests.rest.blacklist', [ 'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade', - 'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data', - 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster', + 'mixed_cluster/30_ml_jobs_crud/*', + 'mixed_cluster/40_ml_datafeed_crud/*', ].join(',') finalizedBy "${baseName}#oldClusterTestCluster#node1.stop" } @@ -236,6 +256,11 @@ subprojects { systemProperty 'tests.rest.suite', 'mixed_cluster' systemProperty 'tests.first_round', 'false' finalizedBy "${baseName}#oldClusterTestCluster#node2.stop" + + systemProperty 'tests.rest.blacklist', [ + 'mixed_cluster/30_ml_jobs_crud/*', + 'mixed_cluster/40_ml_datafeed_crud/*', + ].join(',') } Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) @@ -265,6 +290,11 @@ subprojects { systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster' } } + + systemProperty 'tests.rest.blacklist', [ + 'upgraded_cluster/30_ml_jobs_crud/*', + 'upgraded_cluster/40_ml_datafeed_crud/*', + ].join(',') } Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { From 0c343ca2ce28875a2f25f4d46a0453b684f0d7a0 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 14 Nov 2018 17:39:57 +0000 Subject: [PATCH 18/57] [ML] Reimplement established model memory (#35500) This is the 7.0 implementation of a master node service to keep track of the native process memory requirement of each ML job with an associated native process. The new ML memory tracker service works when the whole cluster is upgraded to at least version 6.6. For mixed version clusters the old mechanism of established model memory stored on the job in cluster state was used. This means that the old (and complex) code to keep established model memory up to date on the job object has been removed in 7.0. Forward port of #35263 --- .../client/ml/job/config/Job.java | 31 +- .../client/ml/job/config/JobTests.java | 3 - docs/reference/ml/apis/jobresource.asciidoc | 5 - .../xpack/core/ml/MlMetadata.java | 62 +++- .../xpack/core/ml/job/config/Job.java | 79 +---- .../xpack/core/ml/job/config/JobUpdate.java | 48 +-- .../persistence/ElasticsearchMappings.java | 3 - .../ml/job/results/ReservedFieldNames.java | 1 - .../xpack/core/ml/job/config/JobTests.java | 39 +-- .../core/ml/job/config/JobUpdateTests.java | 3 - .../integration/BasicRenormalizationIT.java | 15 +- .../xpack/ml/integration/DatafeedJobsIT.java | 11 - ...erimResultsDeletedAfterReopeningJobIT.java | 2 +- .../ml/integration/OverallBucketsIT.java | 13 +- .../integration/RestoreModelSnapshotIT.java | 11 - .../xpack/ml/MachineLearning.java | 12 +- .../ml/action/TransportDeleteJobAction.java | 9 +- .../ml/action/TransportOpenJobAction.java | 189 +++++++--- .../xpack/ml/job/JobManager.java | 33 +- .../autodetect/AutodetectProcessManager.java | 3 +- .../output/AutoDetectResultProcessor.java | 138 +------- .../xpack/ml/process/MlMemoryTracker.java | 325 ++++++++++++++++++ .../xpack/ml/MlMetadataTests.java | 13 +- .../action/TransportOpenJobActionTests.java | 39 ++- .../AutodetectResultProcessorIT.java | 4 +- .../integration/MlDistributedFailureIT.java | 89 ++++- .../xpack/ml/integration/TooManyJobsIT.java | 2 - .../AutoDetectResultProcessorTests.java | 99 +----- .../ml/process/MlMemoryTrackerTests.java | 195 +++++++++++ 29 files changed, 919 insertions(+), 557 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java index 13b4dcb955a05..7210aefa98740 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java @@ -57,7 +57,6 @@ public class Job implements ToXContentObject { public static final ParseField DATA_DESCRIPTION = new ParseField("data_description"); public static final ParseField DESCRIPTION = new ParseField("description"); public static final ParseField FINISHED_TIME = new ParseField("finished_time"); - public static final ParseField ESTABLISHED_MODEL_MEMORY = new ParseField("established_model_memory"); public static final ParseField MODEL_PLOT_CONFIG = new ParseField("model_plot_config"); public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days"); public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval"); @@ -82,7 +81,6 @@ public class Job implements ToXContentObject { (p) -> TimeUtil.parseTimeField(p, FINISHED_TIME.getPreferredName()), FINISHED_TIME, ValueType.VALUE); - PARSER.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY); PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG); PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS); PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION); @@ -105,7 +103,6 @@ public class Job implements ToXContentObject { private final String description; private final Date createTime; private final Date finishedTime; - private final Long establishedModelMemory; private final AnalysisConfig analysisConfig; private final AnalysisLimits analysisLimits; private final DataDescription dataDescription; @@ -120,7 +117,7 @@ public class Job implements ToXContentObject { private final Boolean deleting; private Job(String jobId, String jobType, List groups, String description, - Date createTime, Date finishedTime, Long establishedModelMemory, + Date createTime, Date finishedTime, AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map customSettings, @@ -132,7 +129,6 @@ private Job(String jobId, String jobType, List groups, String descriptio this.description = description; this.createTime = createTime; this.finishedTime = finishedTime; - this.establishedModelMemory = establishedModelMemory; this.analysisConfig = analysisConfig; this.analysisLimits = analysisLimits; this.dataDescription = dataDescription; @@ -202,16 +198,6 @@ public Date getFinishedTime() { return finishedTime; } - /** - * The established model memory of the job, or null if model - * memory has not reached equilibrium yet. - * - * @return The established model memory of the job - */ - public Long getEstablishedModelMemory() { - return establishedModelMemory; - } - /** * The analysis configuration object * @@ -304,9 +290,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, finishedTime.getTime()); } - if (establishedModelMemory != null) { - builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -362,7 +345,6 @@ public boolean equals(Object other) { && Objects.equals(this.description, that.description) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.finishedTime, that.finishedTime) - && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) && Objects.equals(this.analysisConfig, that.analysisConfig) && Objects.equals(this.analysisLimits, that.analysisLimits) && Objects.equals(this.dataDescription, that.dataDescription) @@ -379,7 +361,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, establishedModelMemory, + return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting); @@ -405,7 +387,6 @@ public static class Builder { private DataDescription dataDescription; private Date createTime; private Date finishedTime; - private Long establishedModelMemory; private ModelPlotConfig modelPlotConfig; private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; @@ -433,7 +414,6 @@ public Builder(Job job) { this.dataDescription = job.getDataDescription(); this.createTime = job.getCreateTime(); this.finishedTime = job.getFinishedTime(); - this.establishedModelMemory = job.getEstablishedModelMemory(); this.modelPlotConfig = job.getModelPlotConfig(); this.renormalizationWindowDays = job.getRenormalizationWindowDays(); this.backgroundPersistInterval = job.getBackgroundPersistInterval(); @@ -494,11 +474,6 @@ Builder setFinishedTime(Date finishedTime) { return this; } - public Builder setEstablishedModelMemory(Long establishedModelMemory) { - this.establishedModelMemory = establishedModelMemory; - return this; - } - public Builder setDataDescription(DataDescription.Builder description) { dataDescription = Objects.requireNonNull(description, DATA_DESCRIPTION.getPreferredName()).build(); return this; @@ -553,7 +528,7 @@ public Job build() { Objects.requireNonNull(id, "[" + ID.getPreferredName() + "] must not be null"); Objects.requireNonNull(jobType, "[" + JOB_TYPE.getPreferredName() + "] must not be null"); return new Job( - id, jobType, groups, description, createTime, finishedTime, establishedModelMemory, + id, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java index 667932d591231..70e8b4296b0df 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java @@ -125,9 +125,6 @@ public static Job.Builder createRandomizedJobBuilder() { if (randomBoolean()) { builder.setFinishedTime(new Date(randomNonNegativeLong())); } - if (randomBoolean()) { - builder.setEstablishedModelMemory(randomNonNegativeLong()); - } builder.setAnalysisConfig(AnalysisConfigTests.createRandomized()); builder.setAnalysisLimits(AnalysisLimitsTests.createRandomized()); diff --git a/docs/reference/ml/apis/jobresource.asciidoc b/docs/reference/ml/apis/jobresource.asciidoc index e0c314724e762..6ce6e1e39307e 100644 --- a/docs/reference/ml/apis/jobresource.asciidoc +++ b/docs/reference/ml/apis/jobresource.asciidoc @@ -42,11 +42,6 @@ so do not set the `background_persist_interval` value too low. `description`:: (string) An optional description of the job. -`established_model_memory`:: - (long) The approximate amount of memory resources that have been used for - analytical processing. This field is present only when the analytics have used - a stable amount of memory for several consecutive buckets. - `finished_time`:: (string) If the job closed or failed, this is the time the job finished, otherwise it is `null`. This property is informational; you cannot change its diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index afb56dc6cc8c9..e98773a2ce4de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -57,8 +57,9 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom { public static final String TYPE = "ml"; private static final ParseField JOBS_FIELD = new ParseField("jobs"); private static final ParseField DATAFEEDS_FIELD = new ParseField("datafeeds"); + private static final ParseField LAST_MEMORY_REFRESH_VERSION_FIELD = new ParseField("last_memory_refresh_version"); - public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), Collections.emptySortedMap()); + public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), Collections.emptySortedMap(), null); // This parser follows the pattern that metadata is parsed leniently (to allow for enhancements) public static final ObjectParser LENIENT_PARSER = new ObjectParser<>("ml_metadata", true, Builder::new); @@ -66,15 +67,18 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom { LENIENT_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOBS_FIELD); LENIENT_PARSER.declareObjectArray(Builder::putDatafeeds, (p, c) -> DatafeedConfig.LENIENT_PARSER.apply(p, c).build(), DATAFEEDS_FIELD); + LENIENT_PARSER.declareLong(Builder::setLastMemoryRefreshVersion, LAST_MEMORY_REFRESH_VERSION_FIELD); } private final SortedMap jobs; private final SortedMap datafeeds; + private final Long lastMemoryRefreshVersion; private final GroupOrJobLookup groupOrJobLookup; - private MlMetadata(SortedMap jobs, SortedMap datafeeds) { + private MlMetadata(SortedMap jobs, SortedMap datafeeds, Long lastMemoryRefreshVersion) { this.jobs = Collections.unmodifiableSortedMap(jobs); this.datafeeds = Collections.unmodifiableSortedMap(datafeeds); + this.lastMemoryRefreshVersion = lastMemoryRefreshVersion; this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); } @@ -112,6 +116,10 @@ public Set expandDatafeedIds(String expression, boolean allowNoDatafeeds .expand(expression, allowNoDatafeeds); } + public Long getLastMemoryRefreshVersion() { + return lastMemoryRefreshVersion; + } + @Override public Version getMinimalSupportedVersion() { return Version.V_6_0_0_alpha1; @@ -145,7 +153,11 @@ public MlMetadata(StreamInput in) throws IOException { datafeeds.put(in.readString(), new DatafeedConfig(in)); } this.datafeeds = datafeeds; - + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { + lastMemoryRefreshVersion = in.readOptionalLong(); + } else { + lastMemoryRefreshVersion = null; + } this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); } @@ -153,6 +165,9 @@ public MlMetadata(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { writeMap(jobs, out); writeMap(datafeeds, out); + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { + out.writeOptionalLong(lastMemoryRefreshVersion); + } } private static void writeMap(Map map, StreamOutput out) throws IOException { @@ -169,6 +184,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"), params); mapValuesToXContent(JOBS_FIELD, jobs, builder, extendedParams); mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, extendedParams); + if (lastMemoryRefreshVersion != null) { + builder.field(LAST_MEMORY_REFRESH_VERSION_FIELD.getPreferredName(), lastMemoryRefreshVersion); + } return builder; } @@ -185,30 +203,46 @@ public static class MlMetadataDiff implements NamedDiff { final Diff> jobs; final Diff> datafeeds; + final Long lastMemoryRefreshVersion; MlMetadataDiff(MlMetadata before, MlMetadata after) { this.jobs = DiffableUtils.diff(before.jobs, after.jobs, DiffableUtils.getStringKeySerializer()); this.datafeeds = DiffableUtils.diff(before.datafeeds, after.datafeeds, DiffableUtils.getStringKeySerializer()); + this.lastMemoryRefreshVersion = after.lastMemoryRefreshVersion; } public MlMetadataDiff(StreamInput in) throws IOException { this.jobs = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Job::new, MlMetadataDiff::readJobDiffFrom); this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new, - MlMetadataDiff::readSchedulerDiffFrom); + MlMetadataDiff::readDatafeedDiffFrom); + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { + lastMemoryRefreshVersion = in.readOptionalLong(); + } else { + lastMemoryRefreshVersion = null; + } } + /** + * Merge the diff with the ML metadata. + * @param part The current ML metadata. + * @return The new ML metadata. + */ @Override public MetaData.Custom apply(MetaData.Custom part) { TreeMap newJobs = new TreeMap<>(jobs.apply(((MlMetadata) part).jobs)); TreeMap newDatafeeds = new TreeMap<>(datafeeds.apply(((MlMetadata) part).datafeeds)); - return new MlMetadata(newJobs, newDatafeeds); + // lastMemoryRefreshVersion always comes from the diff - no need to merge with the old value + return new MlMetadata(newJobs, newDatafeeds, lastMemoryRefreshVersion); } @Override public void writeTo(StreamOutput out) throws IOException { jobs.writeTo(out); datafeeds.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { + out.writeOptionalLong(lastMemoryRefreshVersion); + } } @Override @@ -220,7 +254,7 @@ static Diff readJobDiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(Job::new, in); } - static Diff readSchedulerDiffFrom(StreamInput in) throws IOException { + static Diff readDatafeedDiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(DatafeedConfig::new, in); } } @@ -233,7 +267,8 @@ public boolean equals(Object o) { return false; MlMetadata that = (MlMetadata) o; return Objects.equals(jobs, that.jobs) && - Objects.equals(datafeeds, that.datafeeds); + Objects.equals(datafeeds, that.datafeeds) && + Objects.equals(lastMemoryRefreshVersion, that.lastMemoryRefreshVersion); } @Override @@ -243,13 +278,14 @@ public final String toString() { @Override public int hashCode() { - return Objects.hash(jobs, datafeeds); + return Objects.hash(jobs, datafeeds, lastMemoryRefreshVersion); } public static class Builder { private TreeMap jobs; private TreeMap datafeeds; + private Long lastMemoryRefreshVersion; public Builder() { jobs = new TreeMap<>(); @@ -263,6 +299,7 @@ public Builder(@Nullable MlMetadata previous) { } else { jobs = new TreeMap<>(previous.jobs); datafeeds = new TreeMap<>(previous.datafeeds); + lastMemoryRefreshVersion = previous.lastMemoryRefreshVersion; } } @@ -382,8 +419,13 @@ private Builder putDatafeeds(Collection datafeeds) { return this; } + public Builder setLastMemoryRefreshVersion(Long lastMemoryRefreshVersion) { + this.lastMemoryRefreshVersion = lastMemoryRefreshVersion; + return this; + } + public MlMetadata build() { - return new MlMetadata(jobs, datafeeds); + return new MlMetadata(jobs, datafeeds, lastMemoryRefreshVersion); } public void markJobAsDeleting(String jobId, PersistentTasksCustomMetaData tasks, boolean allowDeleteOpenJob) { @@ -420,8 +462,6 @@ void checkJobHasNoDatafeed(String jobId) { } } - - public static MlMetadata getMlMetadata(ClusterState state) { MlMetadata mlMetadata = (state == null) ? null : state.getMetaData().custom(TYPE); if (mlMetadata == null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 5b244ba44d5c3..032eef00649dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -67,7 +67,6 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public static final ParseField DATA_DESCRIPTION = new ParseField("data_description"); public static final ParseField DESCRIPTION = new ParseField("description"); public static final ParseField FINISHED_TIME = new ParseField("finished_time"); - public static final ParseField ESTABLISHED_MODEL_MEMORY = new ParseField("established_model_memory"); public static final ParseField MODEL_PLOT_CONFIG = new ParseField("model_plot_config"); public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days"); public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval"); @@ -102,7 +101,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie p -> TimeUtils.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ValueType.VALUE); parser.declareField(Builder::setFinishedTime, p -> TimeUtils.parseTimeField(p, FINISHED_TIME.getPreferredName()), FINISHED_TIME, ValueType.VALUE); - parser.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY); parser.declareObject(Builder::setAnalysisConfig, ignoreUnknownFields ? AnalysisConfig.LENIENT_PARSER : AnalysisConfig.STRICT_PARSER, ANALYSIS_CONFIG); parser.declareObject(Builder::setAnalysisLimits, ignoreUnknownFields ? AnalysisLimits.LENIENT_PARSER : AnalysisLimits.STRICT_PARSER, @@ -140,7 +138,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie // TODO: Use java.time for the Dates here: x-pack-elasticsearch#829 private final Date createTime; private final Date finishedTime; - private final Long establishedModelMemory; private final AnalysisConfig analysisConfig; private final AnalysisLimits analysisLimits; private final DataDescription dataDescription; @@ -156,7 +153,7 @@ private static ObjectParser createParser(boolean ignoreUnknownFie private final boolean deleting; private Job(String jobId, String jobType, Version jobVersion, List groups, String description, - Date createTime, Date finishedTime, Long establishedModelMemory, + Date createTime, Date finishedTime, AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map customSettings, @@ -169,7 +166,6 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.description = description; this.createTime = createTime; this.finishedTime = finishedTime; - this.establishedModelMemory = establishedModelMemory; this.analysisConfig = analysisConfig; this.analysisLimits = analysisLimits; this.dataDescription = dataDescription; @@ -203,10 +199,9 @@ public Job(StreamInput in) throws IOException { in.readVLong(); } } - if (in.getVersion().onOrAfter(Version.V_6_1_0)) { - establishedModelMemory = in.readOptionalLong(); - } else { - establishedModelMemory = null; + // for removed establishedModelMemory field + if (in.getVersion().onOrAfter(Version.V_6_1_0) && in.getVersion().before(Version.V_7_0_0_alpha1)) { + in.readOptionalLong(); } analysisConfig = new AnalysisConfig(in); analysisLimits = in.readOptionalWriteable(AnalysisLimits::new); @@ -314,16 +309,6 @@ public Date getFinishedTime() { return finishedTime; } - /** - * The established model memory of the job, or null if model - * memory has not reached equilibrium yet. - * - * @return The established model memory of the job - */ - public Long getEstablishedModelMemory() { - return establishedModelMemory; - } - /** * The analysis configuration object * @@ -430,21 +415,6 @@ public Collection allInputFields() { return allFields; } - /** - * Make a best estimate of the job's memory footprint using the information available. - * If a job has an established model memory size, then this is the best estimate. - * Otherwise, assume the maximum model memory limit will eventually be required. - * In either case, a fixed overhead is added to account for the memory required by the - * program code and stack. - * @return an estimate of the memory requirement of this job, in bytes - */ - public long estimateMemoryFootprint() { - if (establishedModelMemory != null && establishedModelMemory > 0) { - return establishedModelMemory + PROCESS_MEMORY_OVERHEAD.getBytes(); - } - return ByteSizeUnit.MB.toBytes(analysisLimits.getModelMemoryLimit()) + PROCESS_MEMORY_OVERHEAD.getBytes(); - } - /** * Returns the timestamp before which data is not accepted by the job. * This is the latest record timestamp minus the job latency. @@ -487,8 +457,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().before(Version.V_7_0_0_alpha1)) { out.writeBoolean(false); } - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeOptionalLong(establishedModelMemory); + // for removed establishedModelMemory field + if (out.getVersion().onOrAfter(Version.V_6_1_0) && out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeOptionalLong(null); } analysisConfig.writeTo(out); out.writeOptionalWriteable(analysisLimits); @@ -539,9 +510,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, finishedTime.getTime()); } - if (establishedModelMemory != null) { - builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -598,7 +566,6 @@ public boolean equals(Object other) { && Objects.equals(this.description, that.description) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.finishedTime, that.finishedTime) - && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) && Objects.equals(this.analysisConfig, that.analysisConfig) && Objects.equals(this.analysisLimits, that.analysisLimits) && Objects.equals(this.dataDescription, that.dataDescription) @@ -616,7 +583,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, establishedModelMemory, + return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting); @@ -657,7 +624,6 @@ public static class Builder implements Writeable, ToXContentObject { private DataDescription dataDescription; private Date createTime; private Date finishedTime; - private Long establishedModelMemory; private ModelPlotConfig modelPlotConfig; private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; @@ -687,7 +653,6 @@ public Builder(Job job) { this.dataDescription = job.getDataDescription(); this.createTime = job.getCreateTime(); this.finishedTime = job.getFinishedTime(); - this.establishedModelMemory = job.getEstablishedModelMemory(); this.modelPlotConfig = job.getModelPlotConfig(); this.renormalizationWindowDays = job.getRenormalizationWindowDays(); this.backgroundPersistInterval = job.getBackgroundPersistInterval(); @@ -718,8 +683,9 @@ public Builder(StreamInput in) throws IOException { in.readVLong(); } } - if (in.getVersion().onOrAfter(Version.V_6_1_0)) { - establishedModelMemory = in.readOptionalLong(); + // for removed establishedModelMemory field + if (in.getVersion().onOrAfter(Version.V_6_1_0) && in.getVersion().before(Version.V_7_0_0_alpha1)) { + in.readOptionalLong(); } analysisConfig = in.readOptionalWriteable(AnalysisConfig::new); analysisLimits = in.readOptionalWriteable(AnalysisLimits::new); @@ -803,11 +769,6 @@ public Builder setFinishedTime(Date finishedTime) { return this; } - public Builder setEstablishedModelMemory(Long establishedModelMemory) { - this.establishedModelMemory = establishedModelMemory; - return this; - } - public Builder setDataDescription(DataDescription.Builder description) { dataDescription = ExceptionsHelper.requireNonNull(description, DATA_DESCRIPTION.getPreferredName()).build(); return this; @@ -913,8 +874,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().before(Version.V_7_0_0_alpha1)) { out.writeBoolean(false); } - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeOptionalLong(establishedModelMemory); + // for removed establishedModelMemory field + if (out.getVersion().onOrAfter(Version.V_6_1_0) && out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeOptionalLong(null); } out.writeOptionalWriteable(analysisConfig); out.writeOptionalWriteable(analysisLimits); @@ -957,9 +919,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (finishedTime != null) { builder.field(FINISHED_TIME.getPreferredName(), finishedTime.getTime()); } - if (establishedModelMemory != null) { - builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory); - } if (analysisConfig != null) { builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); } @@ -1020,7 +979,6 @@ public boolean equals(Object o) { && Objects.equals(this.dataDescription, that.dataDescription) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.finishedTime, that.finishedTime) - && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) && Objects.equals(this.modelPlotConfig, that.modelPlotConfig) && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) @@ -1036,7 +994,7 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription, - createTime, finishedTime, establishedModelMemory, modelPlotConfig, renormalizationWindowDays, + createTime, finishedTime, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting); } @@ -1112,11 +1070,6 @@ private void validateGroups() { public Job build(Date createTime) { setCreateTime(createTime); setJobVersion(Version.CURRENT); - // TODO: Maybe we _could_ accept a value for this supplied at create time - it would - // mean cloned jobs that hadn't been edited much would start with an accurate expected size. - // But on the other hand it would mean jobs that were cloned and then completely changed - // would start with a size that was completely wrong. - setEstablishedModelMemory(null); return build(); } @@ -1152,7 +1105,7 @@ public Job build() { } return new Job( - id, jobType, jobVersion, groups, description, createTime, finishedTime, establishedModelMemory, + id, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 44e20846f9aa3..5d0b56dd79575 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -57,7 +57,6 @@ public class JobUpdate implements Writeable, ToXContentObject { } // These fields should not be set by a REST request INTERNAL_PARSER.declareString(Builder::setModelSnapshotId, Job.MODEL_SNAPSHOT_ID); - INTERNAL_PARSER.declareLong(Builder::setEstablishedModelMemory, Job.ESTABLISHED_MODEL_MEMORY); INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION); INTERNAL_PARSER.declareBoolean(Builder::setClearFinishTime, CLEAR_JOB_FINISH_TIME); @@ -77,7 +76,6 @@ public class JobUpdate implements Writeable, ToXContentObject { private final Map customSettings; private final String modelSnapshotId; private final Version modelSnapshotMinVersion; - private final Long establishedModelMemory; private final Version jobVersion; private final Boolean clearJobFinishTime; @@ -87,8 +85,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, @Nullable Long modelSnapshotRetentionDays, @Nullable List categorisationFilters, @Nullable Map customSettings, @Nullable String modelSnapshotId, - @Nullable Version modelSnapshotMinVersion, @Nullable Long establishedModelMemory, - @Nullable Version jobVersion, @Nullable Boolean clearJobFinishTime) { + @Nullable Version modelSnapshotMinVersion, @Nullable Version jobVersion, @Nullable Boolean clearJobFinishTime) { this.jobId = jobId; this.groups = groups; this.description = description; @@ -103,7 +100,6 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.customSettings = customSettings; this.modelSnapshotId = modelSnapshotId; this.modelSnapshotMinVersion = modelSnapshotMinVersion; - this.establishedModelMemory = establishedModelMemory; this.jobVersion = jobVersion; this.clearJobFinishTime = clearJobFinishTime; } @@ -135,10 +131,9 @@ public JobUpdate(StreamInput in) throws IOException { } customSettings = in.readMap(); modelSnapshotId = in.readOptionalString(); - if (in.getVersion().onOrAfter(Version.V_6_1_0)) { - establishedModelMemory = in.readOptionalLong(); - } else { - establishedModelMemory = null; + // was establishedModelMemory + if (in.getVersion().onOrAfter(Version.V_6_1_0) && in.getVersion().before(Version.V_7_0_0_alpha1)) { + in.readOptionalLong(); } if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.readBoolean()) { jobVersion = Version.readVersion(in); @@ -181,8 +176,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeOptionalLong(establishedModelMemory); + // was establishedModelMemory + if (out.getVersion().onOrAfter(Version.V_6_1_0) && out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeOptionalLong(null); } if (out.getVersion().onOrAfter(Version.V_6_3_0)) { if (jobVersion != null) { @@ -261,10 +257,6 @@ public Version getModelSnapshotMinVersion() { return modelSnapshotMinVersion; } - public Long getEstablishedModelMemory() { - return establishedModelMemory; - } - public Version getJobVersion() { return jobVersion; } @@ -320,9 +312,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelSnapshotMinVersion != null) { builder.field(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); } - if (establishedModelMemory != null) { - builder.field(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory); - } if (jobVersion != null) { builder.field(Job.JOB_VERSION.getPreferredName(), jobVersion); } @@ -374,9 +363,6 @@ public Set getUpdateFields() { if (modelSnapshotMinVersion != null) { updateFields.add(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName()); } - if (establishedModelMemory != null) { - updateFields.add(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName()); - } if (jobVersion != null) { updateFields.add(Job.JOB_VERSION.getPreferredName()); } @@ -452,14 +438,6 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { if (modelSnapshotMinVersion != null) { builder.setModelSnapshotMinVersion(modelSnapshotMinVersion); } - if (establishedModelMemory != null) { - // An established model memory of zero means we don't actually know the established model memory - if (establishedModelMemory > 0) { - builder.setEstablishedModelMemory(establishedModelMemory); - } else { - builder.setEstablishedModelMemory(null); - } - } if (jobVersion != null) { builder.setJobVersion(jobVersion); } @@ -487,7 +465,6 @@ && updatesDetectors(job) == false && (customSettings == null || Objects.equals(customSettings, job.getCustomSettings())) && (modelSnapshotId == null || Objects.equals(modelSnapshotId, job.getModelSnapshotId())) && (modelSnapshotMinVersion == null || Objects.equals(modelSnapshotMinVersion, job.getModelSnapshotMinVersion())) - && (establishedModelMemory == null || Objects.equals(establishedModelMemory, job.getEstablishedModelMemory())) && (jobVersion == null || Objects.equals(jobVersion, job.getJobVersion())) && ((clearJobFinishTime == null || clearJobFinishTime == false) || job.getFinishedTime() == null); } @@ -536,7 +513,6 @@ public boolean equals(Object other) { && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) - && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) && Objects.equals(this.jobVersion, that.jobVersion) && Objects.equals(this.clearJobFinishTime, that.clearJobFinishTime); } @@ -545,7 +521,7 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion, clearJobFinishTime); + modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime); } public static class DetectorUpdate implements Writeable, ToXContentObject { @@ -655,7 +631,6 @@ public static class Builder { private Map customSettings; private String modelSnapshotId; private Version modelSnapshotMinVersion; - private Long establishedModelMemory; private Version jobVersion; private Boolean clearJobFinishTime; @@ -738,11 +713,6 @@ public Builder setModelSnapshotMinVersion(String modelSnapshotMinVersion) { return this; } - public Builder setEstablishedModelMemory(Long establishedModelMemory) { - this.establishedModelMemory = establishedModelMemory; - return this; - } - public Builder setJobVersion(Version version) { this.jobVersion = version; return this; @@ -761,7 +731,7 @@ public Builder setClearFinishTime(boolean clearJobFinishTime) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion, clearJobFinishTime); + modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 491be55049c10..70cbf1c088249 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -282,9 +282,6 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio .startObject(Job.FINISHED_TIME.getPreferredName()) .field(TYPE, DATE) .endObject() - .startObject(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName()) - .field(TYPE, LONG) // TODO should be ByteSizeValue - .endObject() .startObject(Job.MODEL_PLOT_CONFIG.getPreferredName()) .startObject(PROPERTIES) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 400fcc07f4c53..6926ecb98e892 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -188,7 +188,6 @@ public final class ReservedFieldNames { Job.DATA_DESCRIPTION.getPreferredName(), Job.DESCRIPTION.getPreferredName(), Job.FINISHED_TIME.getPreferredName(), - Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), Job.MODEL_PLOT_CONFIG.getPreferredName(), Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(), Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 55b84995d58f7..13c433d758165 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -436,10 +436,9 @@ public void testBuilder_withInvalidIndexNameThrows() { public void testBuilder_buildWithCreateTime() { Job.Builder builder = buildJobBuilder("foo"); Date now = new Date(); - Job job = builder.setEstablishedModelMemory(randomNonNegativeLong()).build(now); + Job job = builder.build(now); assertEquals(now, job.getCreateTime()); assertEquals(Version.CURRENT, job.getJobVersion()); - assertNull(job.getEstablishedModelMemory()); } public void testJobWithoutVersion() throws IOException { @@ -514,39 +513,6 @@ public void testInvalidGroup_matchesJobId() { assertEquals(e.getMessage(), "job and group names must be unique but job [foo] and group [foo] have the same name"); } - public void testEstimateMemoryFootprint_GivenEstablished() { - Job.Builder builder = buildJobBuilder("established"); - long establishedModelMemory = randomIntBetween(10_000, 2_000_000_000); - builder.setEstablishedModelMemory(establishedModelMemory); - if (randomBoolean()) { - builder.setAnalysisLimits(new AnalysisLimits(randomNonNegativeLong(), null)); - } - assertEquals(establishedModelMemory + Job.PROCESS_MEMORY_OVERHEAD.getBytes(), builder.build().estimateMemoryFootprint()); - } - - public void testEstimateMemoryFootprint_GivenLimitAndNotEstablished() { - Job.Builder builder = buildJobBuilder("limit"); - if (rarely()) { - // An "established" model memory of 0 means "not established". Generally this won't be set, so getEstablishedModelMemory() - // will return null, but if it returns 0 we shouldn't estimate the job's memory requirement to be 0. - builder.setEstablishedModelMemory(0L); - } - ByteSizeValue limit = new ByteSizeValue(randomIntBetween(100, 10000), ByteSizeUnit.MB); - builder.setAnalysisLimits(new AnalysisLimits(limit.getMb(), null)); - assertEquals(limit.getBytes() + Job.PROCESS_MEMORY_OVERHEAD.getBytes(), builder.build().estimateMemoryFootprint()); - } - - public void testEstimateMemoryFootprint_GivenNoLimitAndNotEstablished() { - Job.Builder builder = buildJobBuilder("nolimit"); - if (rarely()) { - // An "established" model memory of 0 means "not established". Generally this won't be set, so getEstablishedModelMemory() - // will return null, but if it returns 0 we shouldn't estimate the job's memory requirement to be 0. - builder.setEstablishedModelMemory(0L); - } - assertEquals(ByteSizeUnit.MB.toBytes(AnalysisLimits.PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB) - + Job.PROCESS_MEMORY_OVERHEAD.getBytes(), builder.build().estimateMemoryFootprint()); - } - public void testEarliestValidTimestamp_GivenEmptyDataCounts() { assertThat(createRandomizedJob().earliestValidTimestamp(new DataCounts("foo")), equalTo(0L)); } @@ -618,9 +584,6 @@ public static Job createRandomizedJob() { if (randomBoolean()) { builder.setFinishedTime(new Date(randomNonNegativeLong())); } - if (randomBoolean()) { - builder.setEstablishedModelMemory(randomNonNegativeLong()); - } builder.setAnalysisConfig(AnalysisConfigTests.createRandomized()); builder.setAnalysisLimits(AnalysisLimits.validateAndSetDefaults(AnalysisLimitsTests.createRandomized(), null, AnalysisLimits.DEFAULT_MODEL_MEMORY_LIMIT_MB)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index c6eb42038901b..e249b22a4a896 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -90,9 +90,6 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (useInternalParser && randomBoolean()) { update.setModelSnapshotMinVersion(Version.CURRENT); } - if (useInternalParser && randomBoolean()) { - update.setEstablishedModelMemory(randomNonNegativeLong()); - } if (useInternalParser && randomBoolean()) { update.setJobVersion(randomFrom(Version.CURRENT, Version.V_6_2_0, Version.V_6_1_0)); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java index cc5a9f4f1b469..d1decd4387f8f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java @@ -6,13 +6,11 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.junit.After; @@ -29,7 +27,7 @@ public class BasicRenormalizationIT extends MlNativeAutodetectIntegTestCase { @After - public void tearDownData() throws Exception { + public void tearDownData() { cleanUp(); } @@ -52,15 +50,6 @@ public void testDefaultRenormalization() throws Exception { // This is the key assertion: if renormalization never happened then the record_score would // be the same as the initial_record_score on the anomaly record that happened earlier assertThat(earlierRecord.getInitialRecordScore(), greaterThan(earlierRecord.getRecordScore())); - - // Since this job ran for 50 buckets, it's a good place to assert - // that established model memory matches model memory in the job stats - assertBusy(() -> { - GetJobsStatsAction.Response.JobStats jobStats = getJobStats(jobId).get(0); - ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - Job updatedJob = getJob(jobId).get(0); - assertThat(updatedJob.getEstablishedModelMemory(), equalTo(modelSizeStats.getModelBytes())); - }); } public void testRenormalizationDisabled() throws Exception { @@ -94,7 +83,7 @@ private void createAndRunJob(String jobId, Long renormalizationWindow) throws Ex closeJob(job.getId()); } - private Job.Builder buildAndRegisterJob(String jobId, TimeValue bucketSpan, Long renormalizationWindow) throws Exception { + private Job.Builder buildAndRegisterJob(String jobId, TimeValue bucketSpan, Long renormalizationWindow) { Detector.Builder detector = new Detector.Builder("count", null); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Arrays.asList(detector.build())); analysisConfig.setBucketSpan(bucketSpan); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index ccef7c3f2e181..be760909e3ea9 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.KillProcessAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; @@ -25,7 +24,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.junit.After; import java.util.ArrayList; @@ -92,15 +90,6 @@ public void testLookbackOnly() throws Exception { }, 60, TimeUnit.SECONDS); waitUntilJobIsClosed(job.getId()); - - // Since this job ran for 168 buckets, it's a good place to assert - // that established model memory matches model memory in the job stats - assertBusy(() -> { - GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); - ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - Job updatedJob = getJob(job.getId()).get(0); - assertThat(updatedJob.getEstablishedModelMemory(), equalTo(modelSizeStats.getModelBytes())); - }); } public void testRealtime() throws Exception { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java index add0b9e8a93a3..2e39658ab504a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java @@ -33,7 +33,7 @@ public class InterimResultsDeletedAfterReopeningJobIT extends MlNativeAutodetectIntegTestCase { @After - public void cleanUpTest() throws Exception { + public void cleanUpTest() { cleanUp(); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java index fe344bf835991..ec670773f2f7b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java @@ -7,14 +7,12 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetOverallBucketsAction; import org.elasticsearch.xpack.core.ml.action.util.PageParams; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.junit.After; import java.util.ArrayList; @@ -36,7 +34,7 @@ public class OverallBucketsIT extends MlNativeAutodetectIntegTestCase { private static final long BUCKET_SPAN_SECONDS = 3600; @After - public void cleanUpTest() throws Exception { + public void cleanUpTest() { cleanUp(); } @@ -99,15 +97,6 @@ public void test() throws Exception { GetOverallBucketsAction.INSTANCE, filteredOverallBucketsRequest).actionGet(); assertThat(filteredOverallBucketsResponse.getOverallBuckets().count(), equalTo(2L)); } - - // Since this job ran for 3000 buckets, it's a good place to assert - // that established model memory matches model memory in the job stats - assertBusy(() -> { - GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); - ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - Job updatedJob = getJob(job.getId()).get(0); - assertThat(updatedJob.getEstablishedModelMemory(), equalTo(modelSizeStats.getModelBytes())); - }); } private static Map createRecord(long timestamp) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java index d7a2b857bf359..42bfe4dcde301 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java @@ -7,12 +7,10 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.junit.After; @@ -82,15 +80,6 @@ public void test() throws Exception { }); closeJob(job.getId()); - - // Since these jobs ran for 72 buckets, it's a good place to assert - // that established model memory matches model memory in the job stats - assertBusy(() -> { - GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); - ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - Job updatedJob = getJob(job.getId()).get(0); - assertThat(updatedJob.getEstablishedModelMemory(), equalTo(modelSizeStats.getModelBytes())); - }); } private Job.Builder buildAndRegisterJob(String jobId, TimeValue bucketSpan) throws Exception { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index efe932f4b5c6b..d9075c5a46fe5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -180,6 +180,7 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerProcessFactory; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.process.MlMemoryTracker; import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.process.NativeControllerHolder; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; @@ -275,6 +276,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu private final SetOnce autodetectProcessManager = new SetOnce<>(); private final SetOnce datafeedManager = new SetOnce<>(); + private final SetOnce memoryTracker = new SetOnce<>(); public MachineLearning(Settings settings, Path configPath) { this.settings = settings; @@ -415,6 +417,8 @@ public Collection createComponents(Client client, ClusterService cluster this.datafeedManager.set(datafeedManager); MlLifeCycleService mlLifeCycleService = new MlLifeCycleService(environment, clusterService, datafeedManager, autodetectProcessManager); + MlMemoryTracker memoryTracker = new MlMemoryTracker(clusterService, threadPool, jobManager, jobResultsProvider); + this.memoryTracker.set(memoryTracker); // This object's constructor attaches to the license state, so there's no need to retain another reference to it new InvalidLicenseEnforcer(getLicenseState(), threadPool, datafeedManager, autodetectProcessManager); @@ -433,7 +437,8 @@ public Collection createComponents(Client client, ClusterService cluster jobDataCountsPersister, datafeedManager, auditor, - new MlAssignmentNotifier(auditor, clusterService) + new MlAssignmentNotifier(auditor, clusterService), + memoryTracker ); } @@ -444,8 +449,9 @@ public List> getPersistentTasksExecutor(ClusterServic } return Arrays.asList( - new TransportOpenJobAction.OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get()), - new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedManager.get()) + new TransportOpenJobAction.OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get(), + memoryTracker.get()), + new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor( datafeedManager.get()) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index edd60cd4756d0..385e33dfe369c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -68,6 +68,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.process.MlMemoryTracker; import org.elasticsearch.xpack.ml.utils.MlIndicesUtils; import java.util.ArrayList; @@ -93,6 +94,7 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction(); } @@ -210,6 +214,9 @@ private void normalDeleteJob(ParentTaskAssigningClient parentTaskClient, DeleteJ ActionListener listener) { String jobId = request.getJobId(); + // We clean up the memory tracker on delete rather than close as close is not a master node action + memoryTracker.removeJob(jobId); + // Step 4. When the job has been removed from the cluster state, return a response // ------- CheckedConsumer apiResponseHandler = jobDeleted -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 98a4a9039b37a..f827d67c9f7c5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -56,6 +56,7 @@ import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; @@ -68,6 +69,7 @@ import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.process.MlMemoryTracker; import java.io.IOException; import java.util.ArrayList; @@ -94,21 +96,23 @@ To ensure that a subsequent close job call will see that same task status (and s */ public class TransportOpenJobAction extends TransportMasterNodeAction { + private static final PersistentTasksCustomMetaData.Assignment AWAITING_LAZY_ASSIGNMENT = + new PersistentTasksCustomMetaData.Assignment(null, "persistent task is awaiting node assignment."); + private final XPackLicenseState licenseState; private final PersistentTasksService persistentTasksService; private final Client client; private final JobResultsProvider jobResultsProvider; private final JobConfigProvider jobConfigProvider; - - private static final PersistentTasksCustomMetaData.Assignment AWAITING_LAZY_ASSIGNMENT = - new PersistentTasksCustomMetaData.Assignment(null, "persistent task is awaiting node assignment."); + private final MlMemoryTracker memoryTracker; @Inject public TransportOpenJobAction(TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState, ClusterService clusterService, PersistentTasksService persistentTasksService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client, - JobResultsProvider jobResultsProvider, JobConfigProvider jobConfigProvider) { + JobResultsProvider jobResultsProvider, JobConfigProvider jobConfigProvider, + MlMemoryTracker memoryTracker) { super(OpenJobAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, OpenJobAction.Request::new); this.licenseState = licenseState; @@ -116,6 +120,7 @@ public TransportOpenJobAction(TransportService transportService, ThreadPool thre this.client = client; this.jobResultsProvider = jobResultsProvider; this.jobConfigProvider = jobConfigProvider; + this.memoryTracker = memoryTracker; } /** @@ -144,6 +149,7 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j int maxConcurrentJobAllocations, int fallbackMaxNumberOfOpenJobs, int maxMachineMemoryPercent, + MlMemoryTracker memoryTracker, Logger logger) { String resultsIndexName = job != null ? job.getResultsIndexName() : null; List unavailableIndices = verifyIndicesPrimaryShardsAreActive(resultsIndexName, clusterState); @@ -154,10 +160,38 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j return new PersistentTasksCustomMetaData.Assignment(null, reason); } + // Try to allocate jobs according to memory usage, but if that's not possible (maybe due to a mixed version cluster or maybe + // because of some weird OS problem) then fall back to the old mechanism of only considering numbers of assigned jobs + boolean allocateByMemory = true; + + if (memoryTracker.isRecentlyRefreshed() == false) { + + boolean scheduledRefresh = memoryTracker.asyncRefresh(ActionListener.wrap( + acknowledged -> { + if (acknowledged) { + logger.trace("Job memory requirement refresh request completed successfully"); + } else { + logger.warn("Job memory requirement refresh request completed but did not set time in cluster state"); + } + }, + e -> logger.error("Failed to refresh job memory requirements", e) + )); + if (scheduledRefresh) { + String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; + logger.debug(reason); + return new PersistentTasksCustomMetaData.Assignment(null, reason); + } else { + allocateByMemory = false; + logger.warn("Falling back to allocating job [{}] by job counts because a memory requirement refresh could not be scheduled", + jobId); + } + } + List reasons = new LinkedList<>(); long maxAvailableCount = Long.MIN_VALUE; + long maxAvailableMemory = Long.MIN_VALUE; DiscoveryNode minLoadedNodeByCount = null; - + DiscoveryNode minLoadedNodeByMemory = null; PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { Map nodeAttributes = node.getAttributes(); @@ -198,10 +232,9 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j } } - long numberOfAssignedJobs = 0; int numberOfAllocatingJobs = 0; - + long assignedJobMemory = 0; if (persistentTasks != null) { // find all the job tasks assigned to this node Collection> assignedTasks = persistentTasks.findTasks( @@ -232,6 +265,15 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) { // Don't count CLOSED or FAILED jobs, as they don't consume native memory ++numberOfAssignedJobs; + OpenJobAction.JobParams params = (OpenJobAction.JobParams) assignedTask.getParams(); + Long jobMemoryRequirement = memoryTracker.getJobMemoryRequirement(params.getJobId()); + if (jobMemoryRequirement == null) { + allocateByMemory = false; + logger.debug("Falling back to allocating job [{}] by job counts because " + + "the memory requirement for job [{}] was not available", jobId, params.getJobId()); + } else { + assignedJobMemory += jobMemoryRequirement; + } } } } @@ -272,10 +314,62 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j maxAvailableCount = availableCount; minLoadedNodeByCount = node; } + + String machineMemoryStr = nodeAttributes.get(MachineLearning.MACHINE_MEMORY_NODE_ATTR); + long machineMemory = -1; + // TODO: remove leniency and reject the node if the attribute is null in 7.0 + if (machineMemoryStr != null) { + try { + machineMemory = Long.parseLong(machineMemoryStr); + } catch (NumberFormatException e) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " + + MachineLearning.MACHINE_MEMORY_NODE_ATTR + " attribute [" + machineMemoryStr + "] is not a long"; + logger.trace(reason); + reasons.add(reason); + continue; + } + } + + if (allocateByMemory) { + if (machineMemory > 0) { + long maxMlMemory = machineMemory * maxMachineMemoryPercent / 100; + Long estimatedMemoryFootprint = memoryTracker.getJobMemoryRequirement(jobId); + if (estimatedMemoryFootprint != null) { + long availableMemory = maxMlMemory - assignedJobMemory; + if (estimatedMemoryFootprint > availableMemory) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + + "], because this node has insufficient available memory. Available memory for ML [" + maxMlMemory + + "], memory required by existing jobs [" + assignedJobMemory + + "], estimated memory required for this job [" + estimatedMemoryFootprint + "]"; + logger.trace(reason); + reasons.add(reason); + continue; + } + + if (maxAvailableMemory < availableMemory) { + maxAvailableMemory = availableMemory; + minLoadedNodeByMemory = node; + } + } else { + // If we cannot get the job memory requirement, + // fall back to simply allocating by job count + allocateByMemory = false; + logger.debug("Falling back to allocating job [{}] by job counts because its memory requirement was not available", + jobId); + } + } else { + // If we cannot get the available memory on any machine in + // the cluster, fall back to simply allocating by job count + allocateByMemory = false; + logger.debug("Falling back to allocating job [{}] by job counts because machine memory was not available for node [{}]", + jobId, nodeNameAndMlAttributes(node)); + } + } } - if (minLoadedNodeByCount != null) { - logger.debug("selected node [{}] for job [{}]", minLoadedNodeByCount, jobId); - return new PersistentTasksCustomMetaData.Assignment(minLoadedNodeByCount.getId(), ""); + DiscoveryNode minLoadedNode = allocateByMemory ? minLoadedNodeByMemory : minLoadedNodeByCount; + if (minLoadedNode != null) { + logger.debug("selected node [{}] for job [{}]", minLoadedNode, jobId); + return new PersistentTasksCustomMetaData.Assignment(minLoadedNode.getId(), ""); } else { String explanation = String.join("|", reasons); logger.debug("no node selected for job [{}], reasons [{}]", jobId, explanation); @@ -451,41 +545,48 @@ public void onFailure(Exception e) { }; // Start job task - ActionListener establishedMemoryUpdateListener = ActionListener.wrap( - response -> { - persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()), - MlTasks.JOB_TASK_NAME, jobParams, waitForJobToStart); - }, - listener::onFailure + ActionListener memoryRequirementRefreshListener = ActionListener.wrap( + mem -> persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()), MlTasks.JOB_TASK_NAME, jobParams, + waitForJobToStart), + listener::onFailure + ); + + // Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks + ActionListener jobUpdateListener = ActionListener.wrap( + response -> memoryTracker.refreshJobMemoryAndAllOthers(jobParams.getJobId(), memoryRequirementRefreshListener), + listener::onFailure ); - // Update established model memory for pre-6.1 jobs that haven't had it set - // and increase the model memory limit for 6.1 - 6.3 jobs + // Increase the model memory limit for 6.1 - 6.3 jobs ActionListener missingMappingsListener = ActionListener.wrap( response -> { Job job = jobParams.getJob(); if (job != null) { Version jobVersion = job.getJobVersion(); - Long jobEstablishedModelMemory = job.getEstablishedModelMemory(); - if ((jobVersion == null || jobVersion.before(Version.V_6_1_0)) - && (jobEstablishedModelMemory == null || jobEstablishedModelMemory == 0)) { - jobResultsProvider.getEstablishedMemoryUsage(job.getId(), null, null, establishedModelMemory -> { - if (establishedModelMemory != null && establishedModelMemory > 0) { - JobUpdate update = new JobUpdate.Builder(job.getId()) - .setEstablishedModelMemory(establishedModelMemory).build(); - UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(job.getId(), update); - - executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest, - establishedMemoryUpdateListener); - } else { - establishedMemoryUpdateListener.onResponse(null); - } - }, listener::onFailure); - } else { - establishedMemoryUpdateListener.onResponse(null); + if (jobVersion != null && + (jobVersion.onOrAfter(Version.V_6_1_0) && jobVersion.before(Version.V_6_3_0))) { + // Increase model memory limit if < 512MB + if (job.getAnalysisLimits() != null && job.getAnalysisLimits().getModelMemoryLimit() != null && + job.getAnalysisLimits().getModelMemoryLimit() < 512L) { + + long updatedModelMemoryLimit = (long) (job.getAnalysisLimits().getModelMemoryLimit() * 1.3); + AnalysisLimits limits = new AnalysisLimits(updatedModelMemoryLimit, + job.getAnalysisLimits().getCategorizationExamplesLimit()); + + JobUpdate update = new JobUpdate.Builder(job.getId()).setJobVersion(Version.CURRENT) + .setAnalysisLimits(limits).build(); + UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(job.getId(), update); + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest, + jobUpdateListener); + } else { + jobUpdateListener.onResponse(null); + } + } + else { + jobUpdateListener.onResponse(null); } } else { - establishedMemoryUpdateListener.onResponse(null); + jobUpdateListener.onResponse(null); } }, listener::onFailure ); @@ -629,6 +730,7 @@ private void addDocMappingIfMissing(String alias, CheckedSupplier { private final AutodetectProcessManager autodetectProcessManager; + private final MlMemoryTracker memoryTracker; /** * The maximum number of open jobs can be different on each node. However, nodes on older versions @@ -642,9 +744,10 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut private volatile int maxLazyMLNodes; public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterService, - AutodetectProcessManager autodetectProcessManager) { + AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker) { super(MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); this.autodetectProcessManager = autodetectProcessManager; + this.memoryTracker = memoryTracker; this.fallbackMaxNumberOfOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(settings); this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings); @@ -658,11 +761,17 @@ public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterS @Override public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobParams params, ClusterState clusterState) { - PersistentTasksCustomMetaData.Assignment assignment =selectLeastLoadedMlNode(params.getJobId(), params.getJob(), clusterState, - maxConcurrentJobAllocations, fallbackMaxNumberOfOpenJobs, maxMachineMemoryPercent, logger); + PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(params.getJobId(), + params.getJob(), + clusterState, + maxConcurrentJobAllocations, + fallbackMaxNumberOfOpenJobs, + maxMachineMemoryPercent, + memoryTracker, + logger); if (assignment.getExecutorNode() == null) { int numMlNodes = 0; - for(DiscoveryNode node : clusterState.getNodes()) { + for (DiscoveryNode node : clusterState.getNodes()) { if (Boolean.valueOf(node.getAttributes().get(MachineLearning.ML_ENABLED_NODE_ATTR))) { numMlNodes++; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 0e9d71b93b2da..7d05f2d02a126 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -66,7 +66,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -580,25 +579,17 @@ public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionList // Step 1. update the job // ------- - Consumer updateJobHandler = response -> { - JobUpdate update = new JobUpdate.Builder(request.getJobId()) - .setModelSnapshotId(modelSnapshot.getSnapshotId()) - .setEstablishedModelMemory(response) - .build(); - - jobConfigProvider.updateJob(request.getJobId(), update, maxModelMemoryLimit, ActionListener.wrap( - job -> { - auditor.info(request.getJobId(), - Messages.getMessage(Messages.JOB_AUDIT_REVERTED, modelSnapshot.getDescription())); - updateHandler.accept(Boolean.TRUE); - }, - actionListener::onFailure - )); - }; - - // Step 0. Find the appropriate established model memory for the reverted job - // ------- - jobResultsProvider.getEstablishedMemoryUsage(request.getJobId(), modelSizeStats.getTimestamp(), modelSizeStats, updateJobHandler, - actionListener::onFailure); + JobUpdate update = new JobUpdate.Builder(request.getJobId()) + .setModelSnapshotId(modelSnapshot.getSnapshotId()) + .build(); + + jobConfigProvider.updateJob(request.getJobId(), update, maxModelMemoryLimit, ActionListener.wrap( + job -> { + auditor.info(request.getJobId(), + Messages.getMessage(Messages.JOB_AUDIT_REVERTED, modelSnapshot.getDescription())); + updateHandler.accept(Boolean.TRUE); + }, + actionListener::onFailure + )); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 24abeb9d45b47..8b0de68fb582b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -521,8 +521,7 @@ AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodet AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, autodetectParams, autoDetectExecutorService, onProcessCrash(jobTask)); AutoDetectResultProcessor processor = new AutoDetectResultProcessor( - client, auditor, jobId, renormalizer, jobResultsPersister, jobResultsProvider, autodetectParams.modelSizeStats(), - autodetectParams.modelSnapshot() != null); + client, auditor, jobId, renormalizer, jobResultsPersister, autodetectParams.modelSizeStats()); ExecutorService autodetectWorkerExecutor; try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { autodetectWorkerExecutor = createAutodetectExecutorService(autoDetectExecutorService); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java index a60c937209c21..f536b79547736 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -20,9 +20,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; @@ -41,7 +38,6 @@ import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; @@ -56,7 +52,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -85,20 +80,11 @@ public class AutoDetectResultProcessor { private static final Logger LOGGER = LogManager.getLogger(AutoDetectResultProcessor.class); - /** - * This is how far behind real-time we'll update the job with the latest established model memory. - * If more updates are received during the delay period then they'll take precedence. - * As a result there will be at most one update of established model memory per delay period. - */ - private static final TimeValue ESTABLISHED_MODEL_MEMORY_UPDATE_DELAY = TimeValue.timeValueSeconds(5); - private final Client client; private final Auditor auditor; private final String jobId; private final Renormalizer renormalizer; private final JobResultsPersister persister; - private final JobResultsProvider jobResultsProvider; - private final boolean restoredSnapshot; final CountDownLatch completionLatch = new CountDownLatch(1); final Semaphore updateModelSnapshotSemaphore = new Semaphore(1); @@ -112,30 +98,21 @@ public class AutoDetectResultProcessor { * New model size stats are read as the process is running */ private volatile ModelSizeStats latestModelSizeStats; - private volatile Date latestDateForEstablishedModelMemoryCalc; - private volatile long latestEstablishedModelMemory; - private volatile boolean haveNewLatestModelSizeStats; - private Future scheduledEstablishedModelMemoryUpdate; // only accessed in synchronized methods public AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, - JobResultsPersister persister, JobResultsProvider jobResultsProvider, - ModelSizeStats latestModelSizeStats, boolean restoredSnapshot) { - this(client, auditor, jobId, renormalizer, persister, jobResultsProvider, latestModelSizeStats, - restoredSnapshot, new FlushListener()); + JobResultsPersister persister, ModelSizeStats latestModelSizeStats) { + this(client, auditor, jobId, renormalizer, persister, latestModelSizeStats, new FlushListener()); } AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer, - JobResultsPersister persister, JobResultsProvider jobResultsProvider, ModelSizeStats latestModelSizeStats, - boolean restoredSnapshot, FlushListener flushListener) { + JobResultsPersister persister, ModelSizeStats latestModelSizeStats, FlushListener flushListener) { this.client = Objects.requireNonNull(client); this.auditor = Objects.requireNonNull(auditor); this.jobId = Objects.requireNonNull(jobId); this.renormalizer = Objects.requireNonNull(renormalizer); this.persister = Objects.requireNonNull(persister); - this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider); this.flushListener = Objects.requireNonNull(flushListener); this.latestModelSizeStats = Objects.requireNonNull(latestModelSizeStats); - this.restoredSnapshot = restoredSnapshot; } public void process(AutodetectProcess process) { @@ -230,17 +207,7 @@ void processResult(Context context, AutodetectResult result) { // persist after deleting interim results in case the new // results are also interim context.bulkResultsPersister.persistBucket(bucket).executeRequest(); - latestDateForEstablishedModelMemoryCalc = bucket.getTimestamp(); ++bucketCount; - - // if we haven't previously set established model memory, consider trying again after - // a reasonable number of buckets have elapsed since the last model size stats update - long minEstablishedTimespanMs = JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE * bucket.getBucketSpan() * 1000L; - if (haveNewLatestModelSizeStats && latestEstablishedModelMemory == 0 && latestDateForEstablishedModelMemoryCalc.getTime() - > latestModelSizeStats.getTimestamp().getTime() + minEstablishedTimespanMs) { - scheduleEstablishedModelMemoryUpdate(ESTABLISHED_MODEL_MEMORY_UPDATE_DELAY); - haveNewLatestModelSizeStats = false; - } } List records = result.getRecords(); if (records != null && !records.isEmpty()) { @@ -331,15 +298,6 @@ private void processModelSizeStats(Context context, ModelSizeStats modelSizeStat persister.persistModelSizeStats(modelSizeStats); notifyModelMemoryStatusChange(context, modelSizeStats); latestModelSizeStats = modelSizeStats; - latestDateForEstablishedModelMemoryCalc = modelSizeStats.getTimestamp(); - haveNewLatestModelSizeStats = true; - - // This is a crude way to NOT refresh the index and NOT attempt to update established model memory during the first 20 buckets - // because this is when the model size stats are likely to be least stable and lots of updates will be coming through, and - // we'll NEVER consider memory usage to be established during this period - if (restoredSnapshot || bucketCount >= JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) { - scheduleEstablishedModelMemoryUpdate(ESTABLISHED_MODEL_MEMORY_UPDATE_DELAY); - } } private void notifyModelMemoryStatusChange(Context context, ModelSizeStats modelSizeStats) { @@ -366,12 +324,11 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { return; } - Map update = new HashMap<>(); + Map update = new HashMap<>(); update.put(Job.MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshot.getSnapshotId()); update.put(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshot.getMinVersion().toString()); - updateJob(jobId, Collections.singletonMap(Job.MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshot.getSnapshotId()), - new ActionListener() { + updateJob(jobId, update, new ActionListener() { @Override public void onResponse(UpdateResponse updateResponse) { updateModelSnapshotSemaphore.release(); @@ -387,67 +344,11 @@ public void onFailure(Exception e) { }); } - /** - * The purpose of this method is to avoid saturating the cluster state update thread - * when a lookback job is churning through buckets very fast and the memory usage of - * the job is changing regularly. The idea is to only update the established model - * memory associated with the job a few seconds after the new value has been received. - * If more updates are received during the delay period then they simply replace the - * value that originally caused the update to be scheduled. This rate limits cluster - * state updates due to established model memory changing to one per job per delay period. - * (In reality updates will only occur this rapidly during lookback. During real-time - * operation the limit of one model size stats document per bucket will mean there is a - * maximum of one cluster state update per job per bucket, and usually the bucket span - * is 5 minutes or more.) - * @param delay The delay before updating established model memory. - */ - synchronized void scheduleEstablishedModelMemoryUpdate(TimeValue delay) { - - if (scheduledEstablishedModelMemoryUpdate == null) { - try { - scheduledEstablishedModelMemoryUpdate = client.threadPool().schedule(delay, MachineLearning.UTILITY_THREAD_POOL_NAME, - () -> runEstablishedModelMemoryUpdate(false)); - LOGGER.trace("[{}] Scheduled established model memory update to run in [{}]", jobId, delay); - } catch (EsRejectedExecutionException e) { - if (e.isExecutorShutdown()) { - LOGGER.debug("failed to schedule established model memory update; shutting down", e); - } else { - throw e; - } - } - } - } - - /** - * This method is called from two places: - * - From the {@link Future} used for delayed updates - * - When shutting down this result processor - * When shutting down the result processor it's only necessary to do anything - * if an update has been scheduled, but we want to do the update immediately. - * Despite cancelling the scheduled update in this case, it's possible that - * it's already started running, in which case this method will get called - * twice in quick succession. But the second call will do nothing, as - * scheduledEstablishedModelMemoryUpdate will have been reset - * to null by the first call. - */ - private synchronized void runEstablishedModelMemoryUpdate(boolean cancelExisting) { - - if (scheduledEstablishedModelMemoryUpdate != null) { - if (cancelExisting) { - LOGGER.debug("[{}] Bringing forward previously scheduled established model memory update", jobId); - FutureUtils.cancel(scheduledEstablishedModelMemoryUpdate); - } - scheduledEstablishedModelMemoryUpdate = null; - updateEstablishedModelMemoryOnJob(); - } - } - private void onAutodetectClose() { onCloseActionsLatch = new CountDownLatch(1); ActionListener updateListener = ActionListener.wrap( updateResponse -> { - runEstablishedModelMemoryUpdate(true); onCloseActionsLatch.countDown(); }, e -> { @@ -462,35 +363,6 @@ private void onAutodetectClose() { ); } - private void updateEstablishedModelMemoryOnJob() { - - // Copy these before committing writes, so the calculation is done based on committed documents - Date latestBucketTimestamp = latestDateForEstablishedModelMemoryCalc; - ModelSizeStats modelSizeStatsForCalc = latestModelSizeStats; - - // We need to make all results written up to and including these stats available for the established memory calculation - persister.commitResultWrites(jobId); - - jobResultsProvider.getEstablishedMemoryUsage(jobId, latestBucketTimestamp, modelSizeStatsForCalc, establishedModelMemory -> { - if (latestEstablishedModelMemory != establishedModelMemory) { - updateJob(jobId, Collections.singletonMap(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory), - new ActionListener() { - @Override - public void onResponse(UpdateResponse response) { - latestEstablishedModelMemory = establishedModelMemory; - LOGGER.debug("[{}] Updated job with established model memory [{}]", jobId, establishedModelMemory); - } - - @Override - public void onFailure(Exception e) { - LOGGER.error("[" + jobId + "] Failed to update job with new established model memory [" + - establishedModelMemory + "]", e); - } - }); - } - }, e -> LOGGER.error("[" + jobId + "] Failed to calculate established model memory", e)); - } - private void updateJob(String jobId, Map update, ActionListener listener) { UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java new file mode 100644 index 0000000000000..63f0fac27d8e8 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.process; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.LocalNodeMasterListener; +import org.elasticsearch.cluster.ack.AckedRequest; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * This class keeps track of the memory requirement of ML jobs. + * It only functions on the master node - for this reason it should only be used by master node actions. + * The memory requirement for ML jobs can be updated in 3 ways: + * 1. For all open ML jobs (via {@link #asyncRefresh}) + * 2. For all open ML jobs, plus one named ML job that is not open (via {@link #refreshJobMemoryAndAllOthers}) + * 3. For one named ML job (via {@link #refreshJobMemory}) + * In all cases a listener informs the caller when the requested updates are complete. + */ +public class MlMemoryTracker implements LocalNodeMasterListener { + + private static final AckedRequest ACKED_REQUEST = new AckedRequest() { + @Override + public TimeValue ackTimeout() { + return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; + } + + @Override + public TimeValue masterNodeTimeout() { + return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; + } + }; + + private static final Duration RECENT_UPDATE_THRESHOLD = Duration.ofMinutes(1); + + private final Logger logger = LogManager.getLogger(MlMemoryTracker.class); + private final ConcurrentHashMap memoryRequirementByJob = new ConcurrentHashMap<>(); + private final List> fullRefreshCompletionListeners = new ArrayList<>(); + + private final ThreadPool threadPool; + private final ClusterService clusterService; + private final JobManager jobManager; + private final JobResultsProvider jobResultsProvider; + private volatile boolean isMaster; + private volatile Instant lastUpdateTime; + + public MlMemoryTracker(ClusterService clusterService, ThreadPool threadPool, JobManager jobManager, + JobResultsProvider jobResultsProvider) { + this.threadPool = threadPool; + this.clusterService = clusterService; + this.jobManager = jobManager; + this.jobResultsProvider = jobResultsProvider; + clusterService.addLocalNodeMasterListener(this); + } + + @Override + public void onMaster() { + isMaster = true; + logger.trace("ML memory tracker on master"); + } + + @Override + public void offMaster() { + isMaster = false; + logger.trace("ML memory tracker off master"); + memoryRequirementByJob.clear(); + lastUpdateTime = null; + } + + @Override + public String executorName() { + return MachineLearning.UTILITY_THREAD_POOL_NAME; + } + + /** + * Is the information in this object sufficiently up to date + * for valid allocation decisions to be made using it? + */ + public boolean isRecentlyRefreshed() { + Instant localLastUpdateTime = lastUpdateTime; + return localLastUpdateTime != null && localLastUpdateTime.plus(RECENT_UPDATE_THRESHOLD).isAfter(Instant.now()); + } + + /** + * Get the memory requirement for a job. + * This method only works on the master node. + * @param jobId The job ID. + * @return The memory requirement of the job specified by {@code jobId}, + * or null if it cannot be calculated. + */ + public Long getJobMemoryRequirement(String jobId) { + + if (isMaster == false) { + return null; + } + + Long memoryRequirement = memoryRequirementByJob.get(jobId); + if (memoryRequirement != null) { + return memoryRequirement; + } + + return null; + } + + /** + * Remove any memory requirement that is stored for the specified job. + * It doesn't matter if this method is called for a job that doesn't have + * a stored memory requirement. + */ + public void removeJob(String jobId) { + memoryRequirementByJob.remove(jobId); + } + + /** + * Uses a separate thread to refresh the memory requirement for every ML job that has + * a corresponding persistent task. This method only works on the master node. + * @param listener Will be called when the async refresh completes or fails. The + * boolean value indicates whether the cluster state was updated + * with the refresh completion time. (If it was then this will in + * cause the persistent tasks framework to check if any persistent + * tasks are awaiting allocation.) + * @return true if the async refresh is scheduled, and false + * if this is not possible for some reason. + */ + public boolean asyncRefresh(ActionListener listener) { + + if (isMaster) { + try { + ActionListener mlMetaUpdateListener = ActionListener.wrap( + aVoid -> recordUpdateTimeInClusterState(listener), + listener::onFailure + ); + threadPool.executor(executorName()).execute( + () -> refresh(clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE), mlMetaUpdateListener)); + return true; + } catch (EsRejectedExecutionException e) { + logger.debug("Couldn't schedule ML memory update - node might be shutting down", e); + } + } + + return false; + } + + /** + * This refreshes the memory requirement for every ML job that has a corresponding + * persistent task and, in addition, one job that doesn't have a persistent task. + * This method only works on the master node. + * @param jobId The job ID of the job whose memory requirement is to be refreshed + * despite not having a corresponding persistent task. + * @param listener Receives the memory requirement of the job specified by {@code jobId}, + * or null if it cannot be calculated. + */ + public void refreshJobMemoryAndAllOthers(String jobId, ActionListener listener) { + + if (isMaster == false) { + listener.onResponse(null); + return; + } + + PersistentTasksCustomMetaData persistentTasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + refresh(persistentTasks, ActionListener.wrap(aVoid -> refreshJobMemory(jobId, listener), listener::onFailure)); + } + + /** + * This refreshes the memory requirement for every ML job that has a corresponding persistent task. + * It does NOT remove entries for jobs that no longer have a persistent task, because that would + * lead to a race where a job was opened part way through the refresh. (Instead, entries are removed + * when jobs are deleted.) + */ + void refresh(PersistentTasksCustomMetaData persistentTasks, ActionListener onCompletion) { + + synchronized (fullRefreshCompletionListeners) { + fullRefreshCompletionListeners.add(onCompletion); + if (fullRefreshCompletionListeners.size() > 1) { + // A refresh is already in progress, so don't do another + return; + } + } + + ActionListener refreshComplete = ActionListener.wrap(aVoid -> { + lastUpdateTime = Instant.now(); + synchronized (fullRefreshCompletionListeners) { + assert fullRefreshCompletionListeners.isEmpty() == false; + for (ActionListener listener : fullRefreshCompletionListeners) { + listener.onResponse(null); + } + fullRefreshCompletionListeners.clear(); + } + }, onCompletion::onFailure); + + // persistentTasks will be null if there's never been a persistent task created in this cluster + if (persistentTasks == null) { + refreshComplete.onResponse(null); + } else { + List> mlJobTasks = persistentTasks.tasks().stream() + .filter(task -> MlTasks.JOB_TASK_NAME.equals(task.getTaskName())).collect(Collectors.toList()); + iterateMlJobTasks(mlJobTasks.iterator(), refreshComplete); + } + } + + private void recordUpdateTimeInClusterState(ActionListener listener) { + + clusterService.submitStateUpdateTask("ml-memory-last-update-time", + new AckedClusterStateUpdateTask(ACKED_REQUEST, listener) { + @Override + protected Boolean newResponse(boolean acknowledged) { + return acknowledged; + } + + @Override + public ClusterState execute(ClusterState currentState) { + MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(currentState); + MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata); + builder.setLastMemoryRefreshVersion(currentState.getVersion() + 1); + MlMetadata newMlMetadata = builder.build(); + if (newMlMetadata.equals(currentMlMetadata)) { + // Return same reference if nothing has changed + return currentState; + } else { + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMlMetadata).build()); + return newState.build(); + } + } + }); + } + + private void iterateMlJobTasks(Iterator> iterator, + ActionListener refreshComplete) { + if (iterator.hasNext()) { + OpenJobAction.JobParams jobParams = (OpenJobAction.JobParams) iterator.next().getParams(); + refreshJobMemory(jobParams.getJobId(), + ActionListener.wrap(mem -> iterateMlJobTasks(iterator, refreshComplete), refreshComplete::onFailure)); + } else { + refreshComplete.onResponse(null); + } + } + + /** + * Refresh the memory requirement for a single job. + * This method only works on the master node. + * @param jobId The ID of the job to refresh the memory requirement for. + * @param listener Receives the job's memory requirement, or null + * if it cannot be calculated. + */ + public void refreshJobMemory(String jobId, ActionListener listener) { + if (isMaster == false) { + listener.onResponse(null); + return; + } + + try { + jobResultsProvider.getEstablishedMemoryUsage(jobId, null, null, + establishedModelMemoryBytes -> { + if (establishedModelMemoryBytes <= 0L) { + setJobMemoryToLimit(jobId, listener); + } else { + Long memoryRequirementBytes = establishedModelMemoryBytes + Job.PROCESS_MEMORY_OVERHEAD.getBytes(); + memoryRequirementByJob.put(jobId, memoryRequirementBytes); + listener.onResponse(memoryRequirementBytes); + } + }, + e -> { + logger.error("[" + jobId + "] failed to calculate job established model memory requirement", e); + setJobMemoryToLimit(jobId, listener); + } + ); + } catch (Exception e) { + logger.error("[" + jobId + "] failed to calculate job established model memory requirement", e); + setJobMemoryToLimit(jobId, listener); + } + } + + private void setJobMemoryToLimit(String jobId, ActionListener listener) { + jobManager.getJob(jobId, ActionListener.wrap(job -> { + Long memoryLimitMb = job.getAnalysisLimits().getModelMemoryLimit(); + if (memoryLimitMb != null) { + Long memoryRequirementBytes = ByteSizeUnit.MB.toBytes(memoryLimitMb) + Job.PROCESS_MEMORY_OVERHEAD.getBytes(); + memoryRequirementByJob.put(jobId, memoryRequirementBytes); + listener.onResponse(memoryRequirementBytes); + } else { + memoryRequirementByJob.remove(jobId); + listener.onResponse(null); + } + }, e -> { + if (e instanceof ResourceNotFoundException) { + // TODO: does this also happen if the .ml-config index exists but is unavailable? + logger.trace("[{}] job deleted during ML memory update", jobId); + } else { + logger.error("[" + jobId + "] failed to get job during ML memory update", e); + } + memoryRequirementByJob.remove(jobId); + listener.onResponse(null); + })); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java index c7ca2ff805eba..eb58221bf5f35 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java @@ -69,6 +69,9 @@ protected MlMetadata createTestInstance() { builder.putJob(job, false); } } + if (randomBoolean()) { + builder.setLastMemoryRefreshVersion(randomNonNegativeLong()); + } return builder.build(); } @@ -438,8 +441,9 @@ protected MlMetadata mutateInstance(MlMetadata instance) { for (Map.Entry entry : datafeeds.entrySet()) { metadataBuilder.putDatafeed(entry.getValue(), Collections.emptyMap()); } + metadataBuilder.setLastMemoryRefreshVersion(instance.getLastMemoryRefreshVersion()); - switch (between(0, 1)) { + switch (between(0, 2)) { case 0: metadataBuilder.putJob(JobTests.createRandomizedJob(), true); break; @@ -459,6 +463,13 @@ protected MlMetadata mutateInstance(MlMetadata instance) { metadataBuilder.putJob(randomJob, false); metadataBuilder.putDatafeed(datafeedConfig, Collections.emptyMap()); break; + case 2: + if (instance.getLastMemoryRefreshVersion() == null) { + metadataBuilder.setLastMemoryRefreshVersion(randomNonNegativeLong()); + } else { + metadataBuilder.setLastMemoryRefreshVersion(null); + } + break; default: throw new AssertionError("Illegal randomisation branch"); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 32ae117f9fd45..84d2ecaf918f9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -50,7 +50,9 @@ import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.notifications.AuditorField; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.process.MlMemoryTracker; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; +import org.junit.Before; import java.io.IOException; import java.net.InetAddress; @@ -71,6 +73,14 @@ public class TransportOpenJobActionTests extends ESTestCase { + private MlMemoryTracker memoryTracker; + + @Before + public void setup() { + memoryTracker = mock(MlMemoryTracker.class); + when(memoryTracker.isRecentlyRefreshed()).thenReturn(true); + } + public void testValidate_jobMissing() { expectThrows(ResourceNotFoundException.class, () -> TransportOpenJobAction.validate("job_id2", null)); } @@ -125,7 +135,7 @@ public void testSelectLeastLoadedMlNode_byCount() { jobBuilder.setJobVersion(Version.CURRENT); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", jobBuilder.build(), - cs.build(), 2, 10, 30, logger); + cs.build(), 2, 10, 30, memoryTracker, logger); assertEquals("", result.getExplanation()); assertEquals("_node_id3", result.getExecutorNode()); } @@ -161,7 +171,7 @@ public void testSelectLeastLoadedMlNode_maxCapacity() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id0", new ByteSizeValue(150, ByteSizeUnit.MB)).build(new Date()); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id0", job, cs.build(), 2, - maxRunningJobsPerNode, 30, logger); + maxRunningJobsPerNode, 30, memoryTracker, logger); assertNull(result.getExecutorNode()); assertTrue(result.getExplanation().contains("because this node is full. Number of opened jobs [" + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]")); @@ -187,7 +197,7 @@ public void testSelectLeastLoadedMlNode_noMlNodes() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id2", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 2, 10, 30, memoryTracker, logger); assertTrue(result.getExplanation().contains("because this node isn't a ml node")); assertNull(result.getExecutorNode()); } @@ -221,7 +231,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id6", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); ClusterState cs = csBuilder.build(); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 2, 10, 30, memoryTracker, logger); assertEquals("_node_id3", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -231,7 +241,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -242,7 +252,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); assertNull("no node selected, because stale task", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -253,7 +263,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); assertNull("no node selected, because null state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -291,7 +301,7 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); // Allocation won't be possible if the stale failed job is treated as opening - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); assertEquals("_node_id1", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -301,7 +311,7 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 2, 10, 30, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 2, 10, 30, memoryTracker, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -332,7 +342,8 @@ public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() { cs.nodes(nodes); metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, + memoryTracker, logger); assertThat(result.getExplanation(), containsString("because this node does not support jobs of type [incompatible_type]")); assertNull(result.getExecutorNode()); } @@ -362,7 +373,7 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_incompatible_model_snapshot", job, cs.build(), - 2, 10, 30, logger); + 2, 10, 30, memoryTracker, logger); assertThat(result.getExplanation(), containsString( "because the job's model snapshot requires a node of version [6.3.0] or higher")); assertNull(result.getExecutorNode()); @@ -389,7 +400,8 @@ public void testSelectLeastLoadedMlNode_jobWithRulesButNoNodeMeetsRequiredVersio cs.metaData(metaData); Job job = jobWithRules("job_with_rules"); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, memoryTracker, + logger); assertThat(result.getExplanation(), containsString( "because jobs using custom_rules require a node of version [6.4.0] or higher")); assertNull(result.getExecutorNode()); @@ -416,7 +428,8 @@ public void testSelectLeastLoadedMlNode_jobWithRulesAndNodeMeetsRequiredVersion( cs.metaData(metaData); Job job = jobWithRules("job_with_rules"); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, memoryTracker, + logger); assertNotNull(result.getExecutorNode()); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index f1f3ff77c9840..505a2b871da0b 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -89,7 +89,7 @@ public void createComponents() throws Exception { renormalizer = mock(Renormalizer.class); capturedUpdateModelSnapshotOnJobRequests = new ArrayList<>(); resultProcessor = new AutoDetectResultProcessor(client(), auditor, JOB_ID, renormalizer, - new JobResultsPersister(client()), jobResultsProvider, new ModelSizeStats.Builder(JOB_ID).build(), false) { + new JobResultsPersister(client()), new ModelSizeStats.Builder(JOB_ID).build()) { @Override protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { capturedUpdateModelSnapshotOnJobRequests.add(modelSnapshot); @@ -100,7 +100,7 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { } @After - public void deleteJob() throws Exception { + public void deleteJob() { DeleteJobAction.Request request = new DeleteJobAction.Request(JOB_ID); AcknowledgedResponse response = client().execute(DeleteJobAction.INSTANCE, request).actionGet(); assertTrue(response.isAcknowledged()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 2e14289da705e..5e4d8fd06030c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -8,10 +8,13 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; @@ -31,21 +34,32 @@ import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.persistent.PersistentTasksClusterService.needsReassignment; public class MlDistributedFailureIT extends BaseMlIntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + .put(MachineLearning.CONCURRENT_JOB_ALLOCATIONS.getKey(), 4) + .build(); + } + public void testFailOver() throws Exception { internalCluster().ensureAtLeastNumDataNodes(3); ensureStableClusterOnAllNodes(3); @@ -58,8 +72,6 @@ public void testFailOver() throws Exception { }); } - @TestLogging("org.elasticsearch.xpack.ml.action:DEBUG,org.elasticsearch.xpack.persistent:TRACE," + - "org.elasticsearch.xpack.ml.datafeed:TRACE") public void testLoseDedicatedMasterNode() throws Exception { internalCluster().ensureAtMostNumDataNodes(0); logger.info("Starting dedicated master node..."); @@ -136,12 +148,12 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception { // Job state is opened but the job is not assigned to a node (because we just killed the only ML node) GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId); GetJobsStatsAction.Response jobStatsResponse = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet(); - assertEquals(jobStatsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + assertEquals(JobState.OPENED, jobStatsResponse.getResponse().results().get(0).getState()); GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(datafeedId); GetDatafeedsStatsAction.Response datafeedStatsResponse = client().execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest).actionGet(); - assertEquals(datafeedStatsResponse.getResponse().results().get(0).getDatafeedState(), DatafeedState.STARTED); + assertEquals(DatafeedState.STARTED, datafeedStatsResponse.getResponse().results().get(0).getDatafeedState()); // Can't normal stop an unassigned datafeed StopDatafeedAction.Request stopDatafeedRequest = new StopDatafeedAction.Request(datafeedId); @@ -170,6 +182,73 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception { assertTrue(closeJobResponse.isClosed()); } + @TestLogging("org.elasticsearch.xpack.ml.action:TRACE,org.elasticsearch.xpack.ml.process:TRACE") + public void testJobRelocationIsMemoryAware() throws Exception { + + internalCluster().ensureAtLeastNumDataNodes(1); + ensureStableClusterOnAllNodes(1); + + // Open 4 small jobs. Since there is only 1 node in the cluster they'll have to go on that node. + + setupJobWithoutDatafeed("small1", new ByteSizeValue(2, ByteSizeUnit.MB)); + setupJobWithoutDatafeed("small2", new ByteSizeValue(2, ByteSizeUnit.MB)); + setupJobWithoutDatafeed("small3", new ByteSizeValue(2, ByteSizeUnit.MB)); + setupJobWithoutDatafeed("small4", new ByteSizeValue(2, ByteSizeUnit.MB)); + + // Expand the cluster to 3 nodes. The 4 small jobs will stay on the + // same node because we don't rebalance jobs that are happily running. + + internalCluster().ensureAtLeastNumDataNodes(3); + ensureStableClusterOnAllNodes(3); + + // Open a big job. This should go on a different node to the 4 small ones. + + setupJobWithoutDatafeed("big1", new ByteSizeValue(500, ByteSizeUnit.MB)); + + // Stop the current master node - this should be the one with the 4 small jobs on. + + internalCluster().stopCurrentMasterNode(); + ensureStableClusterOnAllNodes(2); + + // If memory requirements are used to reallocate the 4 small jobs (as we expect) then they should + // all reallocate to the same node, that being the one that doesn't have the big job on. If job counts + // are used to reallocate the small jobs then this implies the fallback allocation mechanism has been + // used in a situation we don't want it to be used in, and at least one of the small jobs will be on + // the same node as the big job. (This all relies on xpack.ml.node_concurrent_job_allocations being set + // to at least 4, which we do in the nodeSettings() method.) + + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(MetaData.ALL)).actionGet(); + QueryPage jobStats = statsResponse.getResponse(); + assertNotNull(jobStats); + List smallJobNodes = jobStats.results().stream().filter(s -> s.getJobId().startsWith("small") && s.getNode() != null) + .map(s -> s.getNode().getName()).collect(Collectors.toList()); + List bigJobNodes = jobStats.results().stream().filter(s -> s.getJobId().startsWith("big") && s.getNode() != null) + .map(s -> s.getNode().getName()).collect(Collectors.toList()); + logger.info("small job nodes: " + smallJobNodes + ", big job nodes: " + bigJobNodes); + assertEquals(5, jobStats.count()); + assertEquals(4, smallJobNodes.size()); + assertEquals(1, bigJobNodes.size()); + assertEquals(1L, smallJobNodes.stream().distinct().count()); + assertEquals(1L, bigJobNodes.stream().distinct().count()); + assertNotEquals(smallJobNodes, bigJobNodes); + }); + } + + private void setupJobWithoutDatafeed(String jobId, ByteSizeValue modelMemoryLimit) throws Exception { + Job.Builder job = createFareQuoteJob(jobId, modelMemoryLimit); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job); + client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); + + client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())).actionGet(); + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(JobState.OPENED, statsResponse.getResponse().results().get(0).getState()); + }); + } + private void setupJobAndDatafeed(String jobId, String datafeedId) throws Exception { Job.Builder job = createScheduledJob(jobId); PutJobAction.Request putJobRequest = new PutJobAction.Request(job); @@ -183,7 +262,7 @@ private void setupJobAndDatafeed(String jobId, String datafeedId) throws Excepti assertBusy(() -> { GetJobsStatsAction.Response statsResponse = client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); - assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + assertEquals(JobState.OPENED, statsResponse.getResponse().results().get(0).getState()); }); StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(config.getId(), 0L); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 87aa3c5b926e3..c4150d633a8f0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -123,12 +123,10 @@ public void testLazyNodeValidation() throws Exception { }); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34084") public void testSingleNode() throws Exception { verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 100)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34084") public void testMultipleNodes() throws Exception { verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 100)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java index 05f83f4ae49cc..761cb56fa8804 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; @@ -56,9 +55,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.function.Consumer; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; @@ -85,7 +82,6 @@ public class AutoDetectResultProcessorTests extends ESTestCase { private Auditor auditor; private Renormalizer renormalizer; private JobResultsPersister persister; - private JobResultsProvider jobResultsProvider; private FlushListener flushListener; private AutoDetectResultProcessor processorUnderTest; private ScheduledThreadPoolExecutor executor; @@ -95,9 +91,10 @@ public void setUpMocks() { executor = new ScheduledThreadPoolExecutor(1); client = mock(Client.class); doAnswer(invocation -> { - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new UpdateResponse()); - return null; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new UpdateResponse()); + return null; }).when(client).execute(same(UpdateAction.INSTANCE), any(), any()); threadPool = mock(ThreadPool.class); when(client.threadPool()).thenReturn(threadPool); @@ -113,10 +110,9 @@ public void setUpMocks() { persister = mock(JobResultsPersister.class); when(persister.persistModelSnapshot(any(), any())) .thenReturn(new IndexResponse(new ShardId("ml", "uid", 0), "doc", "1", 0L, 0L, 0L, true)); - jobResultsProvider = mock(JobResultsProvider.class); flushListener = mock(FlushListener.class); - processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister, jobResultsProvider, - new ModelSizeStats.Builder(JOB_ID).setTimestamp(new Date(BUCKET_SPAN_MS)).build(), false, flushListener); + processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister, + new ModelSizeStats.Builder(JOB_ID).setTimestamp(new Date(BUCKET_SPAN_MS)).build(), flushListener); } @After @@ -300,8 +296,6 @@ public void testProcessResult_modelSizeStats() { verify(persister, times(1)).persistModelSizeStats(modelSizeStats); verifyNoMoreInteractions(persister); - // No interactions with the jobResultsProvider confirms that the established memory calculation did not run - verifyNoMoreInteractions(jobResultsProvider, auditor); assertEquals(modelSizeStats, processorUnderTest.modelSizeStats()); } @@ -343,85 +337,6 @@ public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() { verifyNoMoreInteractions(auditor); } - public void testProcessResult_modelSizeStatsAfterManyBuckets() throws Exception { - JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class); - when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder); - when(bulkBuilder.persistBucket(any(Bucket.class))).thenReturn(bulkBuilder); - - // To avoid slowing down the test this is using a delay of 1 nanosecond rather than the 5 seconds used in production - setupScheduleDelayTime(TimeValue.timeValueNanos(1)); - - AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder); - context.deleteInterimRequired = false; - for (int i = 0; i < JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE; ++i) { - AutodetectResult result = mock(AutodetectResult.class); - Bucket bucket = mock(Bucket.class); - when(result.getBucket()).thenReturn(bucket); - processorUnderTest.processResult(context, result); - } - - AutodetectResult result = mock(AutodetectResult.class); - ModelSizeStats modelSizeStats = mock(ModelSizeStats.class); - Date timestamp = new Date(BUCKET_SPAN_MS); - when(modelSizeStats.getTimestamp()).thenReturn(timestamp); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); - processorUnderTest.processResult(context, result); - - // Some calls will be made 1 nanosecond later in a different thread, hence the assertBusy() - assertBusy(() -> { - verify(persister, times(1)).persistModelSizeStats(modelSizeStats); - verify(persister, times(1)).commitResultWrites(JOB_ID); - verifyNoMoreInteractions(persister); - verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(timestamp), - eq(modelSizeStats), any(Consumer.class), any(Consumer.class)); - verifyNoMoreInteractions(jobResultsProvider); - assertEquals(modelSizeStats, processorUnderTest.modelSizeStats()); - }); - } - - public void testProcessResult_manyModelSizeStatsInQuickSuccession() throws Exception { - JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class); - when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder); - when(bulkBuilder.persistBucket(any(Bucket.class))).thenReturn(bulkBuilder); - - setupScheduleDelayTime(TimeValue.timeValueSeconds(1)); - - AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder); - context.deleteInterimRequired = false; - ModelSizeStats modelSizeStats = null; - for (int i = 1; i <= JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE + 5; ++i) { - AutodetectResult result = mock(AutodetectResult.class); - Bucket bucket = mock(Bucket.class); - when(bucket.getTimestamp()).thenReturn(new Date(BUCKET_SPAN_MS * i)); - when(result.getBucket()).thenReturn(bucket); - processorUnderTest.processResult(context, result); - if (i > JobResultsProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) { - result = mock(AutodetectResult.class); - modelSizeStats = mock(ModelSizeStats.class); - when(modelSizeStats.getTimestamp()).thenReturn(new Date(BUCKET_SPAN_MS * i)); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); - processorUnderTest.processResult(context, result); - } - } - - ModelSizeStats lastModelSizeStats = modelSizeStats; - assertNotNull(lastModelSizeStats); - Date lastTimestamp = lastModelSizeStats.getTimestamp(); - - // Some calls will be made 1 second later in a different thread, hence the assertBusy() - assertBusy(() -> { - // All the model size stats should be persisted to the index... - verify(persister, times(5)).persistModelSizeStats(any(ModelSizeStats.class)); - // ...but only the last should trigger an established model memory update - verify(persister, times(1)).commitResultWrites(JOB_ID); - verifyNoMoreInteractions(persister); - verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(JOB_ID), eq(lastTimestamp), eq(lastModelSizeStats), - any(Consumer.class), any(Consumer.class)); - verifyNoMoreInteractions(jobResultsProvider); - assertEquals(lastModelSizeStats, processorUnderTest.modelSizeStats()); - }); - } - public void testProcessResult_modelSnapshot() { JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class); @@ -442,7 +357,7 @@ public void testProcessResult_modelSnapshot() { verifyNoMoreInteractions(persister); UpdateRequest capturedRequest = requestCaptor.getValue(); - assertThat(capturedRequest.doc().sourceAsMap().keySet(), contains(Job.MODEL_SNAPSHOT_ID.getPreferredName())); + assertNotNull(capturedRequest.doc().sourceAsMap().get(Job.MODEL_SNAPSHOT_ID.getPreferredName())); } public void testProcessResult_quantiles_givenRenormalizationIsEnabled() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java new file mode 100644 index 0000000000000..cbba7ffa04972 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.process; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; +import org.junit.Before; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MlMemoryTrackerTests extends ESTestCase { + + private ClusterService clusterService; + private ThreadPool threadPool; + private JobManager jobManager; + private JobResultsProvider jobResultsProvider; + private MlMemoryTracker memoryTracker; + + @Before + public void setup() { + + clusterService = mock(ClusterService.class); + threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + Runnable r = (Runnable) invocation.getArguments()[0]; + r.run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(anyString())).thenReturn(executorService); + jobManager = mock(JobManager.class); + jobResultsProvider = mock(JobResultsProvider.class); + memoryTracker = new MlMemoryTracker(clusterService, threadPool, jobManager, jobResultsProvider); + } + + public void testRefreshAll() { + + boolean isMaster = randomBoolean(); + if (isMaster) { + memoryTracker.onMaster(); + } else { + memoryTracker.offMaster(); + } + + int numMlJobTasks = randomIntBetween(2, 5); + Map> tasks = new HashMap<>(); + for (int i = 1; i <= numMlJobTasks; ++i) { + String jobId = "job" + i; + PersistentTasksCustomMetaData.PersistentTask task = makeTestTask(jobId); + tasks.put(task.getId(), task); + } + PersistentTasksCustomMetaData persistentTasks = new PersistentTasksCustomMetaData(numMlJobTasks, tasks); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer listener = (Consumer) invocation.getArguments()[3]; + listener.accept(randomLongBetween(1000, 1000000)); + return null; + }).when(jobResultsProvider).getEstablishedMemoryUsage(anyString(), any(), any(), any(Consumer.class), any()); + + memoryTracker.refresh(persistentTasks, ActionListener.wrap(aVoid -> {}, ESTestCase::assertNull)); + + if (isMaster) { + for (int i = 1; i <= numMlJobTasks; ++i) { + String jobId = "job" + i; + verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(jobId), any(), any(), any(), any()); + } + } else { + verify(jobResultsProvider, never()).getEstablishedMemoryUsage(anyString(), any(), any(), any(), any()); + } + } + + public void testRefreshOne() { + + boolean isMaster = randomBoolean(); + if (isMaster) { + memoryTracker.onMaster(); + } else { + memoryTracker.offMaster(); + } + + String jobId = "job"; + boolean haveEstablishedModelMemory = randomBoolean(); + + long modelBytes = 1024 * 1024; + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer listener = (Consumer) invocation.getArguments()[3]; + listener.accept(haveEstablishedModelMemory ? modelBytes : 0L); + return null; + }).when(jobResultsProvider).getEstablishedMemoryUsage(eq(jobId), any(), any(), any(Consumer.class), any()); + + long modelMemoryLimitMb = 2; + Job job = mock(Job.class); + when(job.getAnalysisLimits()).thenReturn(new AnalysisLimits(modelMemoryLimitMb, 4L)); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(job); + return null; + }).when(jobManager).getJob(eq(jobId), any(ActionListener.class)); + + AtomicReference refreshedMemoryRequirement = new AtomicReference<>(); + memoryTracker.refreshJobMemory(jobId, ActionListener.wrap(refreshedMemoryRequirement::set, ESTestCase::assertNull)); + + if (isMaster) { + if (haveEstablishedModelMemory) { + assertEquals(Long.valueOf(modelBytes + Job.PROCESS_MEMORY_OVERHEAD.getBytes()), + memoryTracker.getJobMemoryRequirement(jobId)); + } else { + assertEquals(Long.valueOf(ByteSizeUnit.MB.toBytes(modelMemoryLimitMb) + Job.PROCESS_MEMORY_OVERHEAD.getBytes()), + memoryTracker.getJobMemoryRequirement(jobId)); + } + } else { + assertNull(memoryTracker.getJobMemoryRequirement(jobId)); + } + + assertEquals(memoryTracker.getJobMemoryRequirement(jobId), refreshedMemoryRequirement.get()); + + memoryTracker.removeJob(jobId); + assertNull(memoryTracker.getJobMemoryRequirement(jobId)); + } + + @SuppressWarnings("unchecked") + public void testRecordUpdateTimeInClusterState() { + + boolean isMaster = randomBoolean(); + if (isMaster) { + memoryTracker.onMaster(); + } else { + memoryTracker.offMaster(); + } + + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + + AtomicReference updateVersion = new AtomicReference<>(); + + doAnswer(invocation -> { + AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocation.getArguments()[1]; + ClusterState currentClusterState = ClusterState.EMPTY_STATE; + ClusterState newClusterState = task.execute(currentClusterState); + assertThat(currentClusterState, not(equalTo(newClusterState))); + MlMetadata newMlMetadata = MlMetadata.getMlMetadata(newClusterState); + updateVersion.set(newMlMetadata.getLastMemoryRefreshVersion()); + task.onAllNodesAcked(null); + return null; + }).when(clusterService).submitStateUpdateTask(anyString(), any(AckedClusterStateUpdateTask.class)); + + memoryTracker.asyncRefresh(ActionListener.wrap(ESTestCase::assertTrue, ESTestCase::assertNull)); + + if (isMaster) { + assertNotNull(updateVersion.get()); + } else { + assertNull(updateVersion.get()); + } + } + + private PersistentTasksCustomMetaData.PersistentTask makeTestTask(String jobId) { + return new PersistentTasksCustomMetaData.PersistentTask<>("job-" + jobId, MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId), + 0, PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT); + } +} From 7b5da88f617d2c6a615b5a93773c624062a64843 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 14 Nov 2018 16:35:02 +0000 Subject: [PATCH 19/57] [ML] Need to wait for shards to replicate in distributed test (#35541) Because the cluster was expanded from 1 node to 3 indices would initially start off with 0 replicas. If the original node was killed before auto-expansion to 1 replica was complete then the test would fail because the indices would be unavailable. --- .../xpack/ml/integration/MlDistributedFailureIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 5e4d8fd06030c..4b1ada80f0ef4 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -201,6 +201,10 @@ public void testJobRelocationIsMemoryAware() throws Exception { internalCluster().ensureAtLeastNumDataNodes(3); ensureStableClusterOnAllNodes(3); + // Wait for the cluster to be green - this means the indices have been replicated. + + ensureGreen(".ml-config", ".ml-anomalies-shared", ".ml-notifications"); + // Open a big job. This should go on a different node to the 4 small ones. setupJobWithoutDatafeed("big1", new ByteSizeValue(500, ByteSizeUnit.MB)); From a6f1be2c70194360ab0fa432d186c41db0bcf0f5 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 19 Nov 2018 10:04:31 +0000 Subject: [PATCH 20/57] [ML] DelayedDataCheckConfig index mappings (#35646) --- .../ml/job/persistence/ElasticsearchMappings.java | 11 +++++++++++ .../xpack/core/ml/job/results/ReservedFieldNames.java | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 70cbf1c088249..66ef6e9ae98b4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -355,6 +356,16 @@ public static void addDatafeedConfigFields(XContentBuilder builder) throws IOExc .endObject() .endObject() .endObject() + .startObject(DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName()) + .startObject(PROPERTIES) + .startObject(DelayedDataCheckConfig.ENABLED.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() .startObject(DatafeedConfig.HEADERS.getPreferredName()) .field(ENABLED, false) .endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 6926ecb98e892..2ebf7f9244776 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -7,6 +7,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -250,6 +251,9 @@ public final class ReservedFieldNames { DatafeedConfig.SCRIPT_FIELDS.getPreferredName(), DatafeedConfig.CHUNKING_CONFIG.getPreferredName(), DatafeedConfig.HEADERS.getPreferredName(), + DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName(), + DelayedDataCheckConfig.ENABLED.getPreferredName(), + DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName(), ChunkingConfig.MODE_FIELD.getPreferredName(), ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(), From f048c52d6abe0a65105f6f4566f0cf4c4be0e63c Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 30 Nov 2018 12:55:46 +0000 Subject: [PATCH 21/57] [ML] JIndex: Restore finalize job action (#35939) --- .../xpack/core/ml/job/messages/Messages.java | 3 +- .../ml/action/TransportCloseJobAction.java | 9 +- .../action/TransportDeleteFilterAction.java | 3 +- .../TransportFinalizeJobExecutionAction.java | 55 ++++++++++- .../action/TransportGetJobsStatsAction.java | 2 +- .../output/AutoDetectResultProcessor.java | 96 ++++--------------- ...nsportFinalizeJobExecutionActionTests.java | 89 +++++++++++++++++ .../AutoDetectResultProcessorTests.java | 37 ++----- .../rest-api-spec/test/ml/filter_crud.yml | 2 +- 9 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 710b0eaaf4e53..4039d0be59e0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -48,8 +48,9 @@ public final class Messages { "Datafeed frequency [{0}] must be a multiple of the aggregation interval [{1}]"; public static final String DATAFEED_ID_ALREADY_TAKEN = "A datafeed with id [{0}] already exists"; - public static final String FILTER_NOT_FOUND = "No filter with id [{0}] exists"; + public static final String FILTER_CANNOT_DELETE = "Cannot delete filter [{0}] currently used by jobs {1}"; public static final String FILTER_CONTAINS_TOO_MANY_ITEMS = "Filter [{0}] contains too many items; up to [{1}] items are allowed"; + public static final String FILTER_NOT_FOUND = "No filter with id [{0}] exists"; public static final String INCONSISTENT_ID = "Inconsistent {0}; ''{1}'' specified in the body differs from ''{2}'' specified as a URL argument"; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index b7d8e096f3d23..713b0c43a8c9e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; +import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; @@ -47,6 +48,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + public class TransportCloseJobAction extends TransportTasksAction { @@ -422,7 +426,10 @@ void waitForJobClosed(CloseJobAction.Request request, WaitForCloseRequest waitFo }, request.getCloseTimeout(), new ActionListener() { @Override public void onResponse(Boolean result) { - listener.onResponse(response); + FinalizeJobExecutionAction.Request finalizeRequest = new FinalizeJobExecutionAction.Request( + waitForCloseRequest.jobsToFinalize.toArray(new String[0])); + executeAsyncWithOrigin(client, ML_ORIGIN, FinalizeJobExecutionAction.INSTANCE, finalizeRequest, + ActionListener.wrap(r -> listener.onResponse(response), listener::onFailure)); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java index e552df1b81d99..bc4b25af1feb9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; @@ -58,7 +59,7 @@ protected void doExecute(Task task, DeleteFilterAction.Request request, ActionLi List currentlyUsedBy = findJobsUsingFilter(jobs, filterId); if (!currentlyUsedBy.isEmpty()) { listener.onFailure(ExceptionsHelper.conflictStatusException( - "Cannot delete filter, currently used by jobs: " + currentlyUsedBy)); + Messages.getMessage(Messages.FILTER_CANNOT_DELETE, filterId, currentlyUsedBy))); } else { deleteFilter(filterId, listener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java index 99ce2484f2cd7..d6c03d6c93fbf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java @@ -7,8 +7,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -18,15 +22,31 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.utils.ChainTaskExecutor; + +import java.util.Collections; +import java.util.Date; +import java.util.Map; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class TransportFinalizeJobExecutionAction extends TransportMasterNodeAction { + private final Client client; + @Inject public TransportFinalizeJobExecutionAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Client client) { super(FinalizeJobExecutionAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, FinalizeJobExecutionAction.Request::new); + this.client = client; } @Override @@ -42,9 +62,36 @@ protected AcknowledgedResponse newResponse() { @Override protected void masterOperation(FinalizeJobExecutionAction.Request request, ClusterState state, ActionListener listener) { - // This action is no longer required but needs to be preserved - // in case it is called by an old node in a mixed cluster - listener.onResponse(new AcknowledgedResponse(true)); + String jobIdString = String.join(",", request.getJobIds()); + logger.debug("finalizing jobs [{}]", jobIdString); + + ChainTaskExecutor chainTaskExecutor = new ChainTaskExecutor(threadPool.executor( + MachineLearning.UTILITY_THREAD_POOL_NAME), true); + + Map update = Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date()); + + for (String jobId: request.getJobIds()) { + UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), + ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); + updateRequest.retryOnConflict(3); + updateRequest.doc(update); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + chainTaskExecutor.add(chainedListener -> { + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateAction.INSTANCE, updateRequest, ActionListener.wrap( + updateResponse -> chainedListener.onResponse(null), + chainedListener::onFailure + )); + }); + } + + chainTaskExecutor.execute(ActionListener.wrap( + aVoid -> { + logger.debug("finalized job [{}]", jobIdString); + listener.onResponse(new AcknowledgedResponse(true)); + }, + listener::onFailure + )); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java index 5779555e84f54..54ff7badf13c7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java @@ -69,6 +69,7 @@ public TransportGetJobsStatsAction(TransportService transportService, @Override protected void doExecute(Task task, GetJobsStatsAction.Request request, ActionListener finalListener) { + logger.debug("Get stats for job [{}]", request.getJobId()); jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), true, ActionListener.wrap( expandedIds -> { @@ -105,7 +106,6 @@ protected QueryPage readTaskResponse(Strea protected void taskOperation(GetJobsStatsAction.Request request, TransportOpenJobAction.JobTask task, ActionListener> listener) { String jobId = task.getJobId(); - logger.debug("Get stats for job [{}]", jobId); ClusterState state = clusterService.state(); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); Optional> stats = processManager.getStatistics(task); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java index f536b79547736..56bf487efa26d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -11,20 +11,16 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.action.update.UpdateAction; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xpack.core.ml.MachineLearningField; -import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.action.PutJobAction; +import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; @@ -36,7 +32,6 @@ import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; @@ -44,12 +39,8 @@ import org.elasticsearch.xpack.ml.notifications.Auditor; import java.time.Duration; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; @@ -88,7 +79,6 @@ public class AutoDetectResultProcessor { final CountDownLatch completionLatch = new CountDownLatch(1); final Semaphore updateModelSnapshotSemaphore = new Semaphore(1); - volatile CountDownLatch onCloseActionsLatch; private final FlushListener flushListener; private volatile boolean processKilled; private volatile boolean failed; @@ -149,18 +139,8 @@ public void process(AutodetectProcess process) { } catch (Exception e) { LOGGER.warn(new ParameterizedMessage("[{}] Error persisting autodetect results", jobId), e); } - if (processKilled == false) { - try { - onAutodetectClose(); - } catch (Exception e) { - if (onCloseActionsLatch != null) { - onCloseActionsLatch.countDown(); - } - throw e; - } - } - LOGGER.info("[{}] {} buckets parsed from autodetect output", jobId, bucketCount); + } catch (Exception e) { failed = true; @@ -313,6 +293,9 @@ private void notifyModelMemoryStatusChange(Context context, ModelSizeStats model } protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { + JobUpdate update = new JobUpdate.Builder(jobId).setModelSnapshotId(modelSnapshot.getSnapshotId()).build(); + UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(jobId, update); + try { // This blocks the main processing thread in the unlikely event // there are 2 model snapshots queued up. But it also has the @@ -324,52 +307,20 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) { return; } - Map update = new HashMap<>(); - update.put(Job.MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshot.getSnapshotId()); - update.put(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshot.getMinVersion().toString()); - - updateJob(jobId, update, new ActionListener() { - @Override - public void onResponse(UpdateResponse updateResponse) { - updateModelSnapshotSemaphore.release(); - LOGGER.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId()); - } - - @Override - public void onFailure(Exception e) { - updateModelSnapshotSemaphore.release(); - LOGGER.error("[" + jobId + "] Failed to update job with new model snapshot id [" + - modelSnapshot.getSnapshotId() + "]", e); - } - }); - } - - private void onAutodetectClose() { - onCloseActionsLatch = new CountDownLatch(1); - - ActionListener updateListener = ActionListener.wrap( - updateResponse -> { - onCloseActionsLatch.countDown(); - }, - e -> { - LOGGER.error("[" + jobId + "] Failed to finalize job on autodetect close", e); - onCloseActionsLatch.countDown(); - } - ); - - updateJob(jobId, Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date()), - new ThreadedActionListener<>(LOGGER, client.threadPool(), - MachineLearning.UTILITY_THREAD_POOL_NAME, updateListener, false) - ); - } + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest, new ActionListener() { + @Override + public void onResponse(PutJobAction.Response response) { + updateModelSnapshotSemaphore.release(); + LOGGER.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId()); + } - private void updateJob(String jobId, Map update, ActionListener listener) { - UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId)); - updateRequest.retryOnConflict(3); - updateRequest.doc(update); - updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - executeAsyncWithOrigin(client, ML_ORIGIN, UpdateAction.INSTANCE, updateRequest, listener); + @Override + public void onFailure(Exception e) { + updateModelSnapshotSemaphore.release(); + LOGGER.error("[" + jobId + "] Failed to update job with new model snapshot id [" + + modelSnapshot.getSnapshotId() + "]", e); + } + }); } public void awaitCompletion() throws TimeoutException { @@ -381,13 +332,6 @@ public void awaitCompletion() throws TimeoutException { throw new TimeoutException("Timed out waiting for results processor to complete for job " + jobId); } - // Once completionLatch has passed then onCloseActionsLatch must either - // be set or null, it will not be set later. - if (onCloseActionsLatch != null && onCloseActionsLatch.await( - MachineLearningField.STATE_PERSIST_RESTORE_TIMEOUT.getMinutes(), TimeUnit.MINUTES) == false) { - throw new TimeoutException("Timed out waiting for results processor run post close actions " + jobId); - } - // Input stream has been completely processed at this point. // Wait for any updateModelSnapshotOnJob calls to complete. updateModelSnapshotSemaphore.acquire(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java new file mode 100644 index 0000000000000..fc44c520ebfc7 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.junit.Before; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TransportFinalizeJobExecutionActionTests extends ESTestCase { + + private ThreadPool threadPool; + private Client client; + + @Before + @SuppressWarnings("unchecked") + private void setupMocks() { + ExecutorService executorService = mock(ExecutorService.class); + threadPool = mock(ThreadPool.class); + org.elasticsearch.mock.orig.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); + + client = mock(Client.class); + doAnswer( invocationOnMock -> { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(null); + return null; + }).when(client).execute(eq(UpdateAction.INSTANCE), any(), any()); + + when(client.threadPool()).thenReturn(threadPool); + when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + } + + public void testOperation() { + ClusterService clusterService = mock(ClusterService.class); + TransportFinalizeJobExecutionAction action = createAction(clusterService); + + ClusterState clusterState = ClusterState.builder(new ClusterName("finalize-job-action-tests")).build(); + + FinalizeJobExecutionAction.Request request = new FinalizeJobExecutionAction.Request(new String[]{"job1", "job2"}); + AtomicReference ack = new AtomicReference<>(); + action.masterOperation(request, clusterState, ActionListener.wrap( + ack::set, + e -> assertNull(e.getMessage()) + )); + + assertTrue(ack.get().isAcknowledged()); + verify(client, times(2)).execute(eq(UpdateAction.INSTANCE), any(), any()); + verify(clusterService, never()).submitStateUpdateTask(any(), any()); + } + + private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { + return new TransportFinalizeJobExecutionAction(mock(TransportService.class), clusterService, + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), client); + + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java index 761cb56fa8804..807ac81830904 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java @@ -8,12 +8,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.action.update.UpdateAction; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -23,7 +19,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -34,7 +31,6 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; @@ -42,7 +38,6 @@ import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.After; import org.junit.Before; -import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import java.time.Duration; @@ -51,7 +46,6 @@ import java.util.Date; import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -62,7 +56,6 @@ import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -90,21 +83,9 @@ public class AutoDetectResultProcessorTests extends ESTestCase { public void setUpMocks() { executor = new ScheduledThreadPoolExecutor(1); client = mock(Client.class); - doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new UpdateResponse()); - return null; - }).when(client).execute(same(UpdateAction.INSTANCE), any(), any()); threadPool = mock(ThreadPool.class); when(client.threadPool()).thenReturn(threadPool); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); - ExecutorService executorService = mock(ExecutorService.class); - org.elasticsearch.mock.orig.Mockito.doAnswer(invocation -> { - ((Runnable) invocation.getArguments()[0]).run(); - return null; - }).when(executorService).execute(any(Runnable.class)); - when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); auditor = mock(Auditor.class); renormalizer = mock(Renormalizer.class); persister = mock(JobResultsPersister.class); @@ -133,9 +114,7 @@ public void testProcess() throws TimeoutException { processorUnderTest.process(process); processorUnderTest.awaitCompletion(); verify(renormalizer, times(1)).waitUntilIdle(); - verify(client, times(1)).execute(same(UpdateAction.INSTANCE), any(), any()); assertEquals(0, processorUnderTest.completionLatch.getCount()); - assertEquals(0, processorUnderTest.onCloseActionsLatch.getCount()); } public void testProcessResult_bucket() { @@ -352,12 +331,11 @@ public void testProcessResult_modelSnapshot() { verify(persister, times(1)).persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE); - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(UpdateRequest.class); - verify(client).execute(same(UpdateAction.INSTANCE), requestCaptor.capture(), any()); - verifyNoMoreInteractions(persister); + UpdateJobAction.Request expectedJobUpdateRequest = UpdateJobAction.Request.internal(JOB_ID, + new JobUpdate.Builder(JOB_ID).setModelSnapshotId("a_snapshot_id").build()); - UpdateRequest capturedRequest = requestCaptor.getValue(); - assertNotNull(capturedRequest.doc().sourceAsMap().get(Job.MODEL_SNAPSHOT_ID.getPreferredName())); + verify(client).execute(same(UpdateJobAction.INSTANCE), eq(expectedJobUpdateRequest), any()); + verifyNoMoreInteractions(persister); } public void testProcessResult_quantiles_givenRenormalizationIsEnabled() { @@ -413,7 +391,6 @@ public void testAwaitCompletion() throws TimeoutException { processorUnderTest.awaitCompletion(); assertEquals(0, processorUnderTest.completionLatch.getCount()); assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits()); - assertEquals(0, processorUnderTest.onCloseActionsLatch.getCount()); } public void testPersisterThrowingDoesntBlockProcessing() { @@ -467,7 +444,6 @@ public void testKill() throws TimeoutException { processorUnderTest.process(process); processorUnderTest.awaitCompletion(); - assertNull(processorUnderTest.onCloseActionsLatch); assertEquals(0, processorUnderTest.completionLatch.getCount()); assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits()); @@ -477,7 +453,6 @@ public void testKill() throws TimeoutException { verify(renormalizer).shutdown(); verify(renormalizer, times(1)).waitUntilIdle(); verify(flushListener, times(1)).clear(); - verify(client, never()).execute(same(UpdateAction.INSTANCE), any(), any()); } private void setupScheduleDelayTime(TimeValue delay) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml index 28b5d5c9315e8..fb4b3e764816c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml @@ -246,7 +246,7 @@ setup: } } - do: - catch: conflict + catch: /Cannot delete filter \[filter-foo\] currently used by jobs \[filter-crud\]/ xpack.ml.delete_filter: filter_id: "filter-foo" From fd1e6d464a0a4aa84101df154a98ca25d98f03d1 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 3 Dec 2018 13:48:46 +0000 Subject: [PATCH 22/57] [ML] Replace Version.CURRENT in streaming functions (#36118) --- .../xpack/core/ml/action/OpenJobAction.java | 4 ++-- .../core/ml/action/StartDatafeedAction.java | 4 ++-- .../xpack/core/ml/action/UpdateJobAction.java | 6 ++---- .../xpack/core/ml/job/config/Job.java | 2 +- .../xpack/core/ml/job/config/JobUpdate.java | 16 ++++++++-------- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index 69414c759d7aa..300b55a5e6394 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -174,7 +174,7 @@ public JobParams(String jobId) { public JobParams(StreamInput in) throws IOException { jobId = in.readString(); timeout = TimeValue.timeValueMillis(in.readVLong()); - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { job = in.readOptionalWriteable(Job::new); } } @@ -213,7 +213,7 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); out.writeVLong(timeout.millis()); - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { out.writeOptionalWriteable(job); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index c0f7b1a1229d6..d33280dcac3d2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -195,7 +195,7 @@ public DatafeedParams(StreamInput in) throws IOException { startTime = in.readVLong(); endTime = in.readOptionalLong(); timeout = TimeValue.timeValueMillis(in.readVLong()); - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { jobId = in.readOptionalString(); datafeedIndices = in.readList(StreamInput::readString); } @@ -272,7 +272,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(startTime); out.writeOptionalLong(endTime); out.writeVLong(timeout.millis()); - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { out.writeOptionalString(jobId); out.writeStringList(datafeedIndices); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java index f7e2e514e5769..85e1615c0dfe0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java @@ -97,8 +97,7 @@ public void readFrom(StreamInput in) throws IOException { } else { isInternal = false; } - // TODO jindex change CURRENT to specific version when feature branch is merged - if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.getVersion().before(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.getVersion().before(Version.V_7_0_0)) { in.readBoolean(); // was waitForAck } } @@ -111,8 +110,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_2_2)) { out.writeBoolean(isInternal); } - // TODO jindex change CURRENT to specific version when feature branch is merged - if (out.getVersion().onOrAfter(Version.V_6_3_0) && out.getVersion().before(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_6_3_0) && out.getVersion().before(Version.V_7_0_0)) { out.writeBoolean(false); // was waitForAck } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index c82bec5e920d9..0d9c4093c139e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -809,7 +809,7 @@ public Builder setModelSnapshotMinVersion(Version modelSnapshotMinVersion) { return this; } - public Builder setModelSnapshotMinVersion(String modelSnapshotMinVersion) { + Builder setModelSnapshotMinVersion(String modelSnapshotMinVersion) { this.modelSnapshotMinVersion = Version.fromString(modelSnapshotMinVersion); return this; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 29326f0ddccd0..a0519697e5909 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -140,16 +140,16 @@ public JobUpdate(StreamInput in) throws IOException { } else { jobVersion = null; } + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { + clearJobFinishTime = in.readOptionalBoolean(); + } else { + clearJobFinishTime = null; + } if (in.getVersion().onOrAfter(Version.V_7_0_0) && in.readBoolean()) { modelSnapshotMinVersion = Version.readVersion(in); } else { modelSnapshotMinVersion = null; } - if (in.getVersion().onOrAfter(Version.CURRENT)) { // NORELEASE change current to Jindex release version - clearJobFinishTime = in.readOptionalBoolean(); - } else { - clearJobFinishTime = null; - } } @Override @@ -188,6 +188,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { + out.writeOptionalBoolean(clearJobFinishTime); + } if (out.getVersion().onOrAfter(Version.V_7_0_0)) { if (modelSnapshotMinVersion != null) { out.writeBoolean(true); @@ -196,9 +199,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - if (out.getVersion().onOrAfter(Version.CURRENT)) { // NORELEASE change current to Jindex release version - out.writeOptionalBoolean(clearJobFinishTime); - } } public String getJobId() { From 29fc10dd32215e9f43bcba7663702f12eaa7a557 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 5 Dec 2018 14:38:57 +0000 Subject: [PATCH 23/57] [ML] Use 'anomaly-detector' in job config doc name (#36254) --- .../elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java | 2 +- .../java/org/elasticsearch/xpack/core/ml/job/config/Job.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 4f7eb4b9aa3a9..d4a1f5005e9ea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -219,7 +219,7 @@ public DatafeedConfig(StreamInput in) throws IOException { * @return The ID of document the datafeed config is persisted in */ public static String documentId(String datafeedId) { - return "datafeed-" + datafeedId; + return TYPE + "-" + datafeedId; } public String getId() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 0d9c4093c139e..327c3f7d59e7c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -238,7 +238,7 @@ public static String documentId(String jobId) { throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_ID_TOO_LONG, MlStrings.ID_LENGTH_LIMIT)); } - return "job-" + jobId; + return ANOMALY_DETECTOR_JOB_TYPE + "-" + jobId; } From 81549e66ddb096b2ca8a7e4385a8b0a2b86232e7 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 5 Dec 2018 11:00:36 +0000 Subject: [PATCH 24/57] [ML] Job In Index: Migrate config from the clusterstate (#35834) Migrate ML configuration from clusterstate to index for closed jobs only once all nodes are v6.6.0 or higher --- .../xpack/core/ml/MlMetadata.java | 4 +- .../elasticsearch/xpack/core/ml/MlTasks.java | 17 + .../xpack/core/ml/MlTasksTests.java | 20 + .../xpack/ml/MachineLearning.java | 4 +- .../xpack/ml/MlAssignmentNotifier.java | 42 +- .../xpack/ml/MlConfigMigrator.java | 384 ++++++++++++++++++ .../xpack/ml/MlInitializationService.java | 28 +- .../ml/action/TransportCloseJobAction.java | 9 +- .../ml/action/TransportOpenJobAction.java | 14 +- .../persistence/DatafeedConfigProvider.java | 11 +- .../ml/job/persistence/JobConfigProvider.java | 13 +- .../autodetect/AutodetectProcessManager.java | 14 +- .../xpack/ml/MlAssignmentNotifierTests.java | 103 ++++- .../xpack/ml/MlConfigMigratorTests.java | 222 ++++++++++ .../ml/MlInitializationServiceTests.java | 57 +-- .../ml/integration/MlConfigMigratorIT.java | 143 +++++++ 16 files changed, 980 insertions(+), 105 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index e98773a2ce4de..f0eef3c728841 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -405,14 +405,14 @@ private void checkDatafeedIsStopped(Supplier msg, String datafeedId, Per } } - private Builder putJobs(Collection jobs) { + public Builder putJobs(Collection jobs) { for (Job job : jobs) { putJob(job, true); } return this; } - private Builder putDatafeeds(Collection datafeeds) { + public Builder putDatafeeds(Collection datafeeds) { for (DatafeedConfig datafeed : datafeeds) { this.datafeeds.put(datafeed.getId(), datafeed); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index 5ba04fcc4087c..e78649d152296 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -98,6 +98,23 @@ public static Set openJobIds(@Nullable PersistentTasksCustomMetaData tas .collect(Collectors.toSet()); } + /** + * The datafeed Ids of started datafeed tasks + * + * @param tasks Persistent tasks. If null an empty set is returned. + * @return The Ids of running datafeed tasks + */ + public static Set startedDatafeedIds(@Nullable PersistentTasksCustomMetaData tasks) { + if (tasks == null) { + return Collections.emptySet(); + } + + return tasks.findTasks(DATAFEED_TASK_NAME, task -> true) + .stream() + .map(t -> t.getId().substring(DATAFEED_TASK_ID_PREFIX.length())) + .collect(Collectors.toSet()); + } + /** * Is there an ml anomaly detector job task for the job {@code jobId}? * @param jobId The job id diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index c3579fe4173b8..408520472c4f2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -91,6 +91,26 @@ public void testOpenJobIds_GivenNull() { assertThat(MlTasks.openJobIds(null), empty()); } + public void testStartedDatafeedIds() { + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + assertThat(MlTasks.openJobIds(tasksBuilder.build()), empty()); + + tasksBuilder.addTask(MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.datafeedTaskId("df1"), MlTasks.DATAFEED_TASK_NAME, + new StartDatafeedAction.DatafeedParams("df1", 0L), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.datafeedTaskId("df2"), MlTasks.DATAFEED_TASK_NAME, + new StartDatafeedAction.DatafeedParams("df2", 0L), + new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment")); + + assertThat(MlTasks.startedDatafeedIds(tasksBuilder.build()), containsInAnyOrder("df1", "df2")); + } + + public void testStartedDatafeedIds_GivenNull() { + assertThat(MlTasks.startedDatafeedIds(null), empty()); + } + public void testTaskExistsForJob() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); assertFalse(MlTasks.taskExistsForJob("job-1", tasksBuilder.build())); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index ae319c6cab7ac..77a3f7a2d7489 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -438,7 +438,7 @@ public Collection createComponents(Client client, ClusterService cluster jobDataCountsPersister, datafeedManager, auditor, - new MlAssignmentNotifier(auditor, clusterService), + new MlAssignmentNotifier(auditor, threadPool, client, clusterService), memoryTracker ); } @@ -453,7 +453,7 @@ public List> getPersistentTasksExecutor(ClusterServic return Arrays.asList( new TransportOpenJobAction.OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get(), - memoryTracker.get()), + memoryTracker.get(), client), new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor( datafeedManager.get()) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index 7c82ff6783def..4e6db3fa0bbff 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -7,7 +7,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -30,12 +34,23 @@ public class MlAssignmentNotifier implements ClusterStateListener, LocalNodeMast private final Auditor auditor; private final ClusterService clusterService; - + private final MlConfigMigrator mlConfigMigrator; + private final ThreadPool threadPool; private final AtomicBoolean enabled = new AtomicBoolean(false); - MlAssignmentNotifier(Auditor auditor, ClusterService clusterService) { + MlAssignmentNotifier(Auditor auditor, ThreadPool threadPool, Client client, ClusterService clusterService) { + this.auditor = auditor; + this.clusterService = clusterService; + this.mlConfigMigrator = new MlConfigMigrator(client, clusterService); + this.threadPool = threadPool; + clusterService.addLocalNodeMasterListener(this); + } + + MlAssignmentNotifier(Auditor auditor, ThreadPool threadPool, MlConfigMigrator mlConfigMigrator, ClusterService clusterService) { this.auditor = auditor; this.clusterService = clusterService; + this.mlConfigMigrator = mlConfigMigrator; + this.threadPool = threadPool; clusterService.addLocalNodeMasterListener(this); } @@ -72,6 +87,25 @@ public void clusterChanged(ClusterChangedEvent event) { return; } + Version minNodeVersion = event.state().nodes().getMinNodeVersion(); + if (minNodeVersion.onOrAfter(Version.V_6_6_0)) { + // ok to migrate + mlConfigMigrator.migrateConfigsWithoutTasks(event.state(), ActionListener.wrap( + response -> threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())), + e -> { + logger.error("error migrating ml configurations", e); + threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); + } + )); + } else { + threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); + } + + } + + private void auditChangesToMlTasks(PersistentTasksCustomMetaData current, PersistentTasksCustomMetaData previous, + ClusterState state) { + for (PersistentTask currentTask : current.tasks()) { Assignment currentAssignment = currentTask.getAssignment(); PersistentTask previousTask = previous != null ? previous.getTask(currentTask.getId()) : null; @@ -84,7 +118,7 @@ public void clusterChanged(ClusterChangedEvent event) { if (currentAssignment.getExecutorNode() == null) { auditor.warning(jobId, "No node found to open job. Reasons [" + currentAssignment.getExplanation() + "]"); } else { - DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); + DiscoveryNode node = state.nodes().get(currentAssignment.getExecutorNode()); auditor.info(jobId, "Opening job on node [" + node.toString() + "]"); } } else if (MlTasks.DATAFEED_TASK_NAME.equals(currentTask.getTaskName())) { @@ -98,7 +132,7 @@ public void clusterChanged(ClusterChangedEvent event) { auditor.warning(jobId, msg); } } else { - DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); + DiscoveryNode node = state.nodes().get(currentAssignment.getExecutorNode()); if (jobId != null) { auditor.info(jobId, "Starting datafeed [" + datafeedParams.getDatafeedId() + "] on node [" + node + "]"); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java new file mode 100644 index 0000000000000..22dc43b9326fb --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -0,0 +1,384 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +/** + * Migrates job and datafeed configurations from the clusterstate to + * index documents. + * + * There are 3 steps to the migration process + * 1. Read config from the clusterstate + * - If a job or datafeed is added after this call it will be added to the index + * - If deleted then it's possible the config will be copied before it is deleted. + * Mitigate against this by filtering out jobs marked as deleting + * 2. Copy the config to the index + * - The index operation could fail, don't delete from clusterstate in this case + * 3. Remove config from the clusterstate + * - Before this happens config is duplicated in index and clusterstate, all ops + * must prefer to use the index config at this stage + * - If the clusterstate update fails then the config will remain duplicated + * and the migration process should try again + * + * If there was an error in step 3 and the config is in both the clusterstate and + * index then when the migrator retries it must not overwrite an existing job config + * document as once the index document is present all update operations will function + * on that rather than the clusterstate + */ +public class MlConfigMigrator { + + private static final Logger logger = LogManager.getLogger(MlConfigMigrator.class); + + public static final String MIGRATED_FROM_VERSION = "migrated from version"; + + private final Client client; + private final ClusterService clusterService; + + private final AtomicBoolean migrationInProgress; + + public MlConfigMigrator(Client client, ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + this.migrationInProgress = new AtomicBoolean(false); + } + + /** + * Migrate ml job and datafeed configurations from the clusterstate + * to index documents. + * + * Configs to be migrated are read from the cluster state then bulk + * indexed into .ml-config. Those successfully indexed are then removed + * from the clusterstate. + * + * Migrated jobs have the job version set to v6.6.0 and the custom settings + * map has an entry added recording the fact the job was migrated and its + * original version e.g. + * "migrated from version" : v6.1.0 + * + * + * @param clusterState The current clusterstate + * @param listener The success listener + */ + public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener listener) { + + if (migrationInProgress.compareAndSet(false, true) == false) { + listener.onResponse(Boolean.FALSE); + return; + } + + Collection datafeedsToMigrate = stoppedDatafeedConfigs(clusterState); + List jobsToMigrate = nonDeletingJobs(closedJobConfigs(clusterState)).stream() + .map(MlConfigMigrator::updateJobForMigration) + .collect(Collectors.toList()); + + ActionListener unMarkMigrationInProgress = ActionListener.wrap( + response -> { + migrationInProgress.set(false); + listener.onResponse(response); + }, + e -> { + migrationInProgress.set(false); + listener.onFailure(e); + } + ); + + if (datafeedsToMigrate.isEmpty() && jobsToMigrate.isEmpty()) { + unMarkMigrationInProgress.onResponse(Boolean.FALSE); + return; + } + + writeConfigToIndex(datafeedsToMigrate, jobsToMigrate, ActionListener.wrap( + failedDocumentIds -> { + List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, jobsToMigrate); + List successfulDatafeedWrites = + filterFailedDatafeedConfigWrites(failedDocumentIds, datafeedsToMigrate); + removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, unMarkMigrationInProgress); + }, + unMarkMigrationInProgress::onFailure + )); + } + + // Exposed for testing + public void writeConfigToIndex(Collection datafeedsToMigrate, + Collection jobsToMigrate, + ActionListener> listener) { + + BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + addJobIndexRequests(jobsToMigrate, bulkRequestBuilder); + addDatafeedIndexRequests(datafeedsToMigrate, bulkRequestBuilder); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, bulkRequestBuilder.request(), + ActionListener.wrap( + bulkResponse -> { + Set failedDocumentIds = documentsNotWritten(bulkResponse); + listener.onResponse(failedDocumentIds); + }, + listener::onFailure), + client::bulk + ); + } + + private void removeFromClusterState(List jobsToRemoveIds, List datafeedsToRemoveIds, + ActionListener listener) { + if (jobsToRemoveIds.isEmpty() && datafeedsToRemoveIds.isEmpty()) { + listener.onResponse(Boolean.FALSE); + return; + } + + AtomicReference removedConfigs = new AtomicReference<>(); + + clusterService.submitStateUpdateTask("remove-migrated-ml-configs", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + RemovalResult removed = removeJobsAndDatafeeds(jobsToRemoveIds, datafeedsToRemoveIds, + MlMetadata.getMlMetadata(currentState)); + removedConfigs.set(removed); + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()) + .putCustom(MlMetadata.TYPE, removed.mlMetadata) + .build()); + return newState.build(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + if (removedConfigs.get() != null) { + if (removedConfigs.get().removedJobIds.isEmpty() == false) { + logger.info("ml job configurations migrated: {}", removedConfigs.get().removedJobIds); + } + if (removedConfigs.get().removedDatafeedIds.isEmpty() == false) { + logger.info("ml datafeed configurations migrated: {}", removedConfigs.get().removedDatafeedIds); + } + } + listener.onResponse(Boolean.TRUE); + } + }); + } + + static class RemovalResult { + MlMetadata mlMetadata; + List removedJobIds; + List removedDatafeedIds; + + RemovalResult(MlMetadata mlMetadata, List removedJobIds, List removedDatafeedIds) { + this.mlMetadata = mlMetadata; + this.removedJobIds = removedJobIds; + this.removedDatafeedIds = removedDatafeedIds; + } + } + + /** + * Remove the datafeeds and jobs listed in the parameters from + * mlMetadata if they exist. An account of removed jobs and datafeeds + * is returned in the result structure alongside a new MlMetadata + * with the config removed. + * + * @param jobsToRemove Jobs + * @param datafeedsToRemove Datafeeds + * @param mlMetadata MlMetadata + * @return Structure tracking which jobs and datafeeds were actually removed + * and the new MlMetadata + */ + static RemovalResult removeJobsAndDatafeeds(List jobsToRemove, List datafeedsToRemove, MlMetadata mlMetadata) { + Map currentJobs = new HashMap<>(mlMetadata.getJobs()); + List removedJobIds = new ArrayList<>(); + for (String jobId : jobsToRemove) { + if (currentJobs.remove(jobId) != null) { + removedJobIds.add(jobId); + } + } + + Map currentDatafeeds = new HashMap<>(mlMetadata.getDatafeeds()); + List removedDatafeedIds = new ArrayList<>(); + for (String datafeedId : datafeedsToRemove) { + if (currentDatafeeds.remove(datafeedId) != null) { + removedDatafeedIds.add(datafeedId); + } + } + + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.setLastMemoryRefreshVersion(mlMetadata.getLastMemoryRefreshVersion()) + .putJobs(currentJobs.values()) + .putDatafeeds(currentDatafeeds.values()); + + return new RemovalResult(builder.build(), removedJobIds, removedDatafeedIds); + } + + private void addJobIndexRequests(Collection jobs, BulkRequestBuilder bulkRequestBuilder) { + ToXContent.Params params = new ToXContent.MapParams(JobConfigProvider.TO_XCONTENT_PARAMS); + for (Job job : jobs) { + bulkRequestBuilder.add(indexRequest(job, Job.documentId(job.getId()), params)); + } + } + + private void addDatafeedIndexRequests(Collection datafeedConfigs, BulkRequestBuilder bulkRequestBuilder) { + ToXContent.Params params = new ToXContent.MapParams(DatafeedConfigProvider.TO_XCONTENT_PARAMS); + for (DatafeedConfig datafeedConfig : datafeedConfigs) { + bulkRequestBuilder.add(indexRequest(datafeedConfig, DatafeedConfig.documentId(datafeedConfig.getId()), params)); + } + } + + private IndexRequest indexRequest(ToXContentObject source, String documentId, ToXContent.Params params) { + IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, documentId); + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + indexRequest.source(source.toXContent(builder, params)); + } catch (IOException e) { + throw new IllegalStateException("failed to serialise object [" + documentId + "]", e); + } + return indexRequest; + } + + public static Job updateJobForMigration(Job job) { + Job.Builder builder = new Job.Builder(job); + Map custom = job.getCustomSettings() == null ? new HashMap<>() : new HashMap<>(job.getCustomSettings()); + custom.put(MIGRATED_FROM_VERSION, job.getJobVersion()); + builder.setCustomSettings(custom); + // Pre v5.5 (ml beta) jobs do not have a version. + // These jobs cannot be opened, we rely on the missing version + // to indicate this. + // See TransportOpenJobAction.validate() + if (job.getJobVersion() != null) { + builder.setJobVersion(Version.CURRENT); + } + return builder.build(); + } + + /** + * Filter jobs marked as deleting from the list of jobs + * are not marked as deleting. + * + * @param jobs The jobs to filter + * @return Jobs not marked as deleting + */ + public static List nonDeletingJobs(List jobs) { + return jobs.stream() + .filter(job -> job.isDeleting() == false) + .collect(Collectors.toList()); + } + + /** + * Find the configurations for all closed jobs in the cluster state. + * Closed jobs are those that do not have an associated persistent task. + * + * @param clusterState The cluster state + * @return The closed job configurations + */ + public static List closedJobConfigs(ClusterState clusterState) { + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + Set openJobIds = MlTasks.openJobIds(persistentTasks); + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + return mlMetadata.getJobs().values().stream() + .filter(job -> openJobIds.contains(job.getId()) == false) + .collect(Collectors.toList()); + } + + /** + * Find the configurations for stopped datafeeds in the cluster state. + * Stopped datafeeds are those that do not have an associated persistent task. + * + * @param clusterState The cluster state + * @return The closed job configurations + */ + public static List stoppedDatafeedConfigs(ClusterState clusterState) { + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + Set startedDatafeedIds = MlTasks.startedDatafeedIds(persistentTasks); + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + return mlMetadata.getDatafeeds().values().stream() + .filter(datafeedConfig-> startedDatafeedIds.contains(datafeedConfig.getId()) == false) + .collect(Collectors.toList()); + } + + /** + * Check for failures in the bulk response and return the + * Ids of any documents not written to the index + * + * If the index operation failed because the document already + * exists this is not considered an error. + * + * @param response BulkResponse + * @return The set of document Ids not written by the bulk request + */ + static Set documentsNotWritten(BulkResponse response) { + Set failedDocumentIds = new HashSet<>(); + + for (BulkItemResponse itemResponse : response.getItems()) { + if (itemResponse.isFailed()) { + BulkItemResponse.Failure failure = itemResponse.getFailure(); + failedDocumentIds.add(itemResponse.getFailure().getId()); + logger.info("failed to index ml configuration [" + itemResponse.getFailure().getId() + "], " + + itemResponse.getFailure().getMessage()); + } else { + logger.info("ml configuration [" + itemResponse.getId() + "] indexed"); + } + } + return failedDocumentIds; + } + + static List filterFailedJobConfigWrites(Set failedDocumentIds, List jobs) { + return jobs.stream() + .map(Job::getId) + .filter(id -> failedDocumentIds.contains(Job.documentId(id)) == false) + .collect(Collectors.toList()); + } + + static List filterFailedDatafeedConfigWrites(Set failedDocumentIds, Collection datafeeds) { + return datafeeds.stream() + .map(DatafeedConfig::getId) + .filter(id -> failedDocumentIds.contains(DatafeedConfig.documentId(id)) == false) + .collect(Collectors.toList()); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 1e890b83391d5..5a635aeed8645 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -6,14 +6,12 @@ package org.elasticsearch.xpack.ml; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleListener; -import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; -class MlInitializationService implements ClusterStateListener { +class MlInitializationService implements LocalNodeMasterListener { private final ThreadPool threadPool; private final ClusterService clusterService; @@ -25,21 +23,21 @@ class MlInitializationService implements ClusterStateListener { this.threadPool = threadPool; this.clusterService = clusterService; this.client = client; - clusterService.addListener(this); } @Override - public void clusterChanged(ClusterChangedEvent event) { - if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { - // Wait until the gateway has recovered from disk. - return; - } + public void onMaster() { + installDailyMaintenanceService(); + } - if (event.localNodeMaster()) { - installDailyMaintenanceService(); - } else { - uninstallDailyMaintenanceService(); - } + @Override + public void offMaster() { + uninstallDailyMaintenanceService(); + } + + @Override + public String executorName() { + return ThreadPool.Names.GENERIC; } private void installDailyMaintenanceService() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index 713b0c43a8c9e..b7d8e096f3d23 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -28,7 +28,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; @@ -48,9 +47,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; - public class TransportCloseJobAction extends TransportTasksAction { @@ -426,10 +422,7 @@ void waitForJobClosed(CloseJobAction.Request request, WaitForCloseRequest waitFo }, request.getCloseTimeout(), new ActionListener() { @Override public void onResponse(Boolean result) { - FinalizeJobExecutionAction.Request finalizeRequest = new FinalizeJobExecutionAction.Request( - waitForCloseRequest.jobsToFinalize.toArray(new String[0])); - executeAsyncWithOrigin(client, ML_ORIGIN, FinalizeJobExecutionAction.INSTANCE, finalizeRequest, - ActionListener.wrap(r -> listener.onResponse(response), listener::onFailure)); + listener.onResponse(response); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 74c90c5923ef5..b69c6714f4018 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -53,6 +53,7 @@ import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; @@ -733,6 +734,7 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut private final AutodetectProcessManager autodetectProcessManager; private final MlMemoryTracker memoryTracker; + private final Client client; /** * The maximum number of open jobs can be different on each node. However, nodes on older versions @@ -746,10 +748,12 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut private volatile int maxLazyMLNodes; public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterService, - AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker) { + AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker, + Client client) { super(MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); this.autodetectProcessManager = autodetectProcessManager; this.memoryTracker = memoryTracker; + this.client = client; this.fallbackMaxNumberOfOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(settings); this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings); @@ -811,9 +815,15 @@ protected void nodeOperation(AllocatedPersistentTask task, OpenJobAction.JobPara return; } + String jobId = jobTask.getJobId(); autodetectProcessManager.openJob(jobTask, e2 -> { if (e2 == null) { - task.markAsCompleted(); + FinalizeJobExecutionAction.Request finalizeRequest = new FinalizeJobExecutionAction.Request(new String[]{jobId}); + executeAsyncWithOrigin(client, ML_ORIGIN, FinalizeJobExecutionAction.INSTANCE, finalizeRequest, + ActionListener.wrap( + response -> task.markAsCompleted(), + e -> logger.error("error finalizing job [" + jobId + "]", e) + )); } else { task.markAsFailed(e2); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index 6a0351694f2ff..ccd22e8c9260d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -59,6 +59,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -78,15 +79,15 @@ public class DatafeedConfigProvider { private final Client client; private final NamedXContentRegistry xContentRegistry; - - private static final Map TO_XCONTENT_PARAMS = new HashMap<>(); + public static final Map TO_XCONTENT_PARAMS; static { - TO_XCONTENT_PARAMS.put(ToXContentParams.FOR_INTERNAL_STORAGE, "true"); - TO_XCONTENT_PARAMS.put(ToXContentParams.INCLUDE_TYPE, "true"); + Map modifiable = new HashMap<>(); + modifiable.put(ToXContentParams.FOR_INTERNAL_STORAGE, "true"); + modifiable.put(ToXContentParams.INCLUDE_TYPE, "true"); + TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable); } public DatafeedConfigProvider(Client client, NamedXContentRegistry xContentRegistry) { - this.client = client; this.xContentRegistry = xContentRegistry; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index fc71d14c987f5..cbc3ba9c4c451 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -67,14 +67,17 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -90,6 +93,14 @@ public class JobConfigProvider { private static final Logger logger = LogManager.getLogger(JobConfigProvider.class); + + public static final Map TO_XCONTENT_PARAMS; + static { + Map modifiable = new HashMap<>(); + modifiable.put(ToXContentParams.FOR_INTERNAL_STORAGE, "true"); + TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable); + } + private final Client client; public JobConfigProvider(Client client) { @@ -106,7 +117,7 @@ public JobConfigProvider(Client client) { */ public void putJob(Job job, ActionListener listener) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - XContentBuilder source = job.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder source = job.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, Job.documentId(job.getId())) .setSource(source) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 27d3cca387651..ff946f09cac41 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -403,7 +403,7 @@ public void onFailure(Exception e) { } } - public void openJob(JobTask jobTask, Consumer handler) { + public void openJob(JobTask jobTask, Consumer closeHandler) { String jobId = jobTask.getJobId(); logger.info("Opening job [{}]", jobId); @@ -411,7 +411,7 @@ public void openJob(JobTask jobTask, Consumer handler) { // NORELEASE JIndex. Should not be doing this work on the network thread job -> { if (job.getJobVersion() == null) { - handler.accept(ExceptionsHelper.badRequestException("Cannot open job [" + jobId + closeHandler.accept(ExceptionsHelper.badRequestException("Cannot open job [" + jobId + "] because jobs created prior to version 5.5 are not supported")); return; } @@ -423,7 +423,7 @@ public void openJob(JobTask jobTask, Consumer handler) { threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { - handler.accept(e); + closeHandler.accept(e); } @Override @@ -440,7 +440,7 @@ protected void doRun() { } try { - createProcessAndSetRunning(processContext, job, params, handler); + createProcessAndSetRunning(processContext, job, params, closeHandler); processContext.getAutodetectCommunicator().init(params.modelSnapshot()); setJobState(jobTask, JobState.OPENED); } catch (Exception e1) { @@ -453,17 +453,17 @@ protected void doRun() { .kill(); processByAllocation.remove(jobTask.getAllocationId()); } finally { - setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); + setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1)); } } } }); }, e1 -> { logger.warn("Failed to gather information required to open job [" + jobId + "]", e1); - setJobState(jobTask, JobState.FAILED, e2 -> handler.accept(e1)); + setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1)); }); }, - handler + closeHandler )); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java index 90a1d45f9e15e..f6754ff7d039d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -14,27 +15,60 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.junit.Before; import java.net.InetAddress; import java.util.Collections; +import java.util.concurrent.ExecutorService; import static org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests.addJobTask; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class MlAssignmentNotifierTests extends ESTestCase { + private Auditor auditor; + private ClusterService clusterService; + private ThreadPool threadPool; + private MlConfigMigrator configMigrator; + + @Before + @SuppressWarnings("unchecked") + private void setupMocks() { + auditor = mock(Auditor.class); + clusterService = mock(ClusterService.class); + threadPool = mock(ThreadPool.class); + configMigrator = mock(MlConfigMigrator.class); + threadPool = mock(ThreadPool.class); + + ExecutorService executorService = mock(ExecutorService.class); + org.elasticsearch.mock.orig.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(anyString())).thenReturn(executorService); + + doAnswer(invocation -> { + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(Boolean.TRUE); + return null; + }).when(configMigrator).migrateConfigsWithoutTasks(any(ClusterState.class), any(ActionListener.class)); + } + public void testClusterChanged_info() { - Auditor auditor = mock(Auditor.class); - ClusterService clusterService = mock(ClusterService.class); - MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, clusterService); + MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); notifier.onMaster(); DiscoveryNode node = @@ -53,6 +87,7 @@ public void testClusterChanged_info() { .build(); notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); verify(auditor, times(1)).info(eq("job_id"), any()); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(state), any()); notifier.offMaster(); notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); @@ -60,9 +95,7 @@ public void testClusterChanged_info() { } public void testClusterChanged_warning() { - Auditor auditor = mock(Auditor.class); - ClusterService clusterService = mock(ClusterService.class); - MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, clusterService); + MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); notifier.onMaster(); ClusterState previous = ClusterState.builder(new ClusterName("_name")) @@ -78,10 +111,66 @@ public void testClusterChanged_warning() { .build(); notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); verify(auditor, times(1)).warning(eq("job_id"), any()); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(state), any()); notifier.offMaster(); notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); verifyNoMoreInteractions(auditor); } + public void testClusterChanged_noPersistentTaskChanges() { + MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); + notifier.onMaster(); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask("job_id", null, null, tasksBuilder); + MetaData metaData = MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()).build(); + ClusterState previous = ClusterState.builder(new ClusterName("_name")) + .metaData(metaData) + .build(); + + ClusterState current = ClusterState.builder(new ClusterName("_name")) + .metaData(metaData) + .build(); + + notifier.clusterChanged(new ClusterChangedEvent("_test", current, previous)); + verify(configMigrator, never()).migrateConfigsWithoutTasks(any(), any()); + + notifier.offMaster(); + verify(configMigrator, never()).migrateConfigsWithoutTasks(any(), any()); + } + + public void testMigrateNotTriggered_GivenPre66Nodes() { + MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); + notifier.onMaster(); + + ClusterState previous = ClusterState.builder(new ClusterName("_name")) + .build(); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask("job_id", null, null, tasksBuilder); + MetaData metaData = MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()).build(); + + // mixed 6.5 and 6.6 nodes + ClusterState current = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0)) + .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0))) + .metaData(metaData) + .build(); + + notifier.clusterChanged(new ClusterChangedEvent("_test", current, previous)); + verify(configMigrator, never()).migrateConfigsWithoutTasks(any(), any()); + + current = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_6_0)) + .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0))) + .metaData(metaData) + .build(); + + // all 6.6 nodes + notifier.clusterChanged(new ClusterChangedEvent("_test", current, previous)); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(any(), any()); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java new file mode 100644 index 0000000000000..faaf4425dfb02 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java @@ -0,0 +1,222 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml; + +import org.elasticsearch.Version; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MlConfigMigratorTests extends ESTestCase { + + public void testNonDeletingJobs() { + Job job1 = JobTests.buildJobBuilder("openjob1").build(); + Job job2 = JobTests.buildJobBuilder("openjob2").build(); + Job deletingJob = JobTests.buildJobBuilder("deleting-job").setDeleting(true).build(); + + assertThat(MlConfigMigrator.nonDeletingJobs(Arrays.asList(job1, job2, deletingJob)), containsInAnyOrder(job1, job2)); + } + + public void testClosedJobConfigs() { + Job openJob1 = JobTests.buildJobBuilder("openjob1").build(); + Job openJob2 = JobTests.buildJobBuilder("openjob2").build(); + + MlMetadata.Builder mlMetadata = new MlMetadata.Builder() + .putJob(openJob1, false) + .putJob(openJob2, false) + .putDatafeed(createCompatibleDatafeed(openJob1.getId()), Collections.emptyMap()); + + ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build()) + .putCustom(PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData.builder().build()) + ) + .build(); + + assertThat(MlConfigMigrator.closedJobConfigs(clusterState), containsInAnyOrder(openJob1, openJob2)); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + tasksBuilder.addTask(MlTasks.jobTaskId("openjob1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + + clusterState = ClusterState.builder(new ClusterName("migratortests")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build()) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) + ) + .build(); + + assertThat(MlConfigMigrator.closedJobConfigs(clusterState), containsInAnyOrder(openJob2)); + } + + public void testStoppedDatafeedConfigs() { + Job openJob1 = JobTests.buildJobBuilder("openjob1").build(); + Job openJob2 = JobTests.buildJobBuilder("openjob2").build(); + DatafeedConfig datafeedConfig1 = createCompatibleDatafeed(openJob1.getId()); + DatafeedConfig datafeedConfig2 = createCompatibleDatafeed(openJob2.getId()); + MlMetadata.Builder mlMetadata = new MlMetadata.Builder() + .putJob(openJob1, false) + .putJob(openJob2, false) + .putDatafeed(datafeedConfig1, Collections.emptyMap()) + .putDatafeed(datafeedConfig2, Collections.emptyMap()); + + ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build()) + .putCustom(PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData.builder().build()) + ) + .build(); + + assertThat(MlConfigMigrator.stoppedDatafeedConfigs(clusterState), containsInAnyOrder(datafeedConfig1, datafeedConfig2)); + + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + tasksBuilder.addTask(MlTasks.jobTaskId("openjob1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(MlTasks.datafeedTaskId(datafeedConfig1.getId()), MlTasks.DATAFEED_TASK_NAME, + new StartDatafeedAction.DatafeedParams(datafeedConfig1.getId(), 0L), + new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment")); + + clusterState = ClusterState.builder(new ClusterName("migratortests")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build()) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) + ) + .build(); + + assertThat(MlConfigMigrator.stoppedDatafeedConfigs(clusterState), containsInAnyOrder(datafeedConfig2)); + } + + public void testUpdateJobForMigration() { + Job.Builder oldJob = JobTests.buildJobBuilder("pre-migration"); + Version oldVersion = Version.V_6_3_0; + oldJob.setJobVersion(oldVersion); + + Job migratedJob = MlConfigMigrator.updateJobForMigration(oldJob.build()); + assertEquals(Version.CURRENT, migratedJob.getJobVersion()); + assertTrue(migratedJob.getCustomSettings().containsKey(MlConfigMigrator.MIGRATED_FROM_VERSION)); + assertEquals(oldVersion, migratedJob.getCustomSettings().get(MlConfigMigrator.MIGRATED_FROM_VERSION)); + } + + public void testUpdateJobForMigration_GivenV54Job() { + Job.Builder oldJob = JobTests.buildJobBuilder("pre-migration"); + // v5.4 jobs did not have a version and should not have a new one set + oldJob.setJobVersion(null); + + Job migratedJob = MlConfigMigrator.updateJobForMigration(oldJob.build()); + assertNull(migratedJob.getJobVersion()); + assertTrue(migratedJob.getCustomSettings().containsKey(MlConfigMigrator.MIGRATED_FROM_VERSION)); + } + + public void testFilterFailedJobConfigWrites() { + List jobs = new ArrayList<>(); + jobs.add(JobTests.buildJobBuilder("foo").build()); + jobs.add(JobTests.buildJobBuilder("bar").build()); + jobs.add(JobTests.buildJobBuilder("baz").build()); + + assertThat(MlConfigMigrator.filterFailedJobConfigWrites(Collections.emptySet(), jobs), hasSize(3)); + assertThat(MlConfigMigrator.filterFailedJobConfigWrites(Collections.singleton(Job.documentId("bar")), jobs), + contains("foo", "baz")); + } + + public void testFilterFailedDatafeedConfigWrites() { + List datafeeds = new ArrayList<>(); + datafeeds.add(createCompatibleDatafeed("foo")); + datafeeds.add(createCompatibleDatafeed("bar")); + datafeeds.add(createCompatibleDatafeed("baz")); + + assertThat(MlConfigMigrator.filterFailedDatafeedConfigWrites(Collections.emptySet(), datafeeds), hasSize(3)); + assertThat(MlConfigMigrator.filterFailedDatafeedConfigWrites(Collections.singleton(DatafeedConfig.documentId("df-foo")), datafeeds), + contains("df-bar", "df-baz")); + } + + public void testDocumentsNotWritten() { + BulkItemResponse ok = mock(BulkItemResponse.class); + when(ok.isFailed()).thenReturn(false); + + BulkItemResponse failed = mock(BulkItemResponse.class); + when(failed.isFailed()).thenReturn(true); + BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); + when(failure.getId()).thenReturn("failed-doc-id"); + when(failure.getCause()).thenReturn(mock(IllegalStateException.class)); + when(failed.getFailure()).thenReturn(failure); + + BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[] {ok, failed}, 1L); + Set docsIds = MlConfigMigrator.documentsNotWritten(bulkResponse); + assertThat(docsIds, contains("failed-doc-id")); + } + + public void testRemoveJobsAndDatafeeds_removeAll() { + Job job1 = JobTests.buildJobBuilder("job1").build(); + Job job2 = JobTests.buildJobBuilder("job2").build(); + DatafeedConfig datafeedConfig1 = createCompatibleDatafeed(job1.getId()); + DatafeedConfig datafeedConfig2 = createCompatibleDatafeed(job2.getId()); + MlMetadata.Builder mlMetadata = new MlMetadata.Builder() + .putJob(job1, false) + .putJob(job2, false) + .putDatafeed(datafeedConfig1, Collections.emptyMap()) + .putDatafeed(datafeedConfig2, Collections.emptyMap()); + + MlConfigMigrator.RemovalResult removalResult = MlConfigMigrator.removeJobsAndDatafeeds( + Arrays.asList("job1", "job2"), Arrays.asList("df-job1", "df-job2"), mlMetadata.build()); + + assertThat(removalResult.mlMetadata.getJobs().keySet(), empty()); + assertThat(removalResult.mlMetadata.getDatafeeds().keySet(), empty()); + assertThat(removalResult.removedJobIds, contains("job1", "job2")); + assertThat(removalResult.removedDatafeedIds, contains("df-job1", "df-job2")); + } + + public void testRemoveJobsAndDatafeeds_removeSome() { + Job job1 = JobTests.buildJobBuilder("job1").build(); + Job job2 = JobTests.buildJobBuilder("job2").build(); + DatafeedConfig datafeedConfig1 = createCompatibleDatafeed(job1.getId()); + MlMetadata.Builder mlMetadata = new MlMetadata.Builder() + .putJob(job1, false) + .putJob(job2, false) + .putDatafeed(datafeedConfig1, Collections.emptyMap()); + + MlConfigMigrator.RemovalResult removalResult = MlConfigMigrator.removeJobsAndDatafeeds( + Arrays.asList("job1", "job-none"), Collections.singletonList("df-none"), mlMetadata.build()); + + assertThat(removalResult.mlMetadata.getJobs().keySet(), contains("job2")); + assertThat(removalResult.mlMetadata.getDatafeeds().keySet(), contains("df-job1")); + assertThat(removalResult.removedJobIds, contains("job1")); + assertThat(removalResult.removedDatafeedIds, empty()); + } + + + private DatafeedConfig createCompatibleDatafeed(String jobId) { + // create a datafeed without aggregations or anything + // else that may cause validation errors + DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("df-" + jobId, jobId); + datafeedBuilder.setIndices(Collections.singletonList("my_index")); + return datafeedBuilder.build(); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index 5ded1b205a110..f96fe79908c15 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -5,22 +5,13 @@ */ package org.elasticsearch.xpack.ml; -import org.elasticsearch.Version; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.junit.Before; -import java.net.InetAddress; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledFuture; @@ -62,46 +53,21 @@ public void setUpMocks() { public void testInitialize() { MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client); - - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) - .localNodeId("_node_id") - .masterNodeId("_node_id")) - .metaData(MetaData.builder()) - .build(); - initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); - + initializationService.onMaster(); assertThat(initializationService.getDailyMaintenanceService().isStarted(), is(true)); } public void testInitialize_noMasterNode() { MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client); - - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT))) - .metaData(MetaData.builder()) - .build(); - initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); - + initializationService.offMaster(); assertThat(initializationService.getDailyMaintenanceService(), is(nullValue())); } public void testInitialize_alreadyInitialized() { MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client); - - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) - .localNodeId("_node_id") - .masterNodeId("_node_id")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, new MlMetadata.Builder().build())) - .build(); MlDailyMaintenanceService initialDailyMaintenanceService = mock(MlDailyMaintenanceService.class); initializationService.setDailyMaintenanceService(initialDailyMaintenanceService); - initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + initializationService.onMaster(); assertSame(initialDailyMaintenanceService, initializationService.getDailyMaintenanceService()); } @@ -111,23 +77,10 @@ public void testNodeGoesFromMasterToNonMasterAndBack() { MlDailyMaintenanceService initialDailyMaintenanceService = mock(MlDailyMaintenanceService.class); initializationService.setDailyMaintenanceService(initialDailyMaintenanceService); - ClusterState masterCs = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) - .localNodeId("_node_id") - .masterNodeId("_node_id")) - .metaData(MetaData.builder()) - .build(); - ClusterState noMasterCs = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT))) - .metaData(MetaData.builder()) - .build(); - initializationService.clusterChanged(new ClusterChangedEvent("_source", noMasterCs, masterCs)); - + initializationService.offMaster(); verify(initialDailyMaintenanceService).stop(); - initializationService.clusterChanged(new ClusterChangedEvent("_source", masterCs, noMasterCs)); + initializationService.onMaster(); MlDailyMaintenanceService finalDailyMaintenanceService = initializationService.getDailyMaintenanceService(); assertNotSame(initialDailyMaintenanceService, finalDailyMaintenanceService); assertThat(initializationService.getDailyMaintenanceService().isStarted(), is(true)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java new file mode 100644 index 0000000000000..51a3b5d2366b0 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +public class MlConfigMigratorIT extends MlSingleNodeTestCase { + + public void testWriteConfigToIndex() throws InterruptedException { + + final String indexJobId = "job-already-migrated"; + // Add a job to the index + JobConfigProvider jobConfigProvider = new JobConfigProvider(client()); + Job indexJob = buildJobBuilder(indexJobId).build(); + // Same as index job but has extra fields in its custom settings + // which will be used to check the config was overwritten + Job migratedJob = MlConfigMigrator.updateJobForMigration(indexJob); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference indexResponseHolder = new AtomicReference<>(); + // put a job representing a previously migrated job + blockingCall(actionListener -> jobConfigProvider.putJob(migratedJob, actionListener), indexResponseHolder, exceptionHolder); + + ClusterService clusterService = mock(ClusterService.class); + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(client(), clusterService); + + AtomicReference> failedIdsHolder = new AtomicReference<>(); + Job foo = buildJobBuilder("foo").build(); + // try to write foo and 'job-already-migrated' which does not have the custom setting field + assertNull(indexJob.getCustomSettings()); + + blockingCall(actionListener -> mlConfigMigrator.writeConfigToIndex(Collections.emptyList(), + Arrays.asList(indexJob, foo), actionListener), + failedIdsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(failedIdsHolder.get(), empty()); + + // Check job foo has been indexed and job-already-migrated has been overwritten + AtomicReference> jobsHolder = new AtomicReference<>(); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, false, actionListener), + jobsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(jobsHolder.get(), hasSize(2)); + Job fooJob = jobsHolder.get().get(0).build(); + assertEquals("foo", fooJob.getId()); + // this job won't have been marked as migrated as calling + // MlConfigMigrator.writeConfigToIndex directly does not do that + assertNull(fooJob.getCustomSettings()); + Job alreadyMigratedJob = jobsHolder.get().get(1).build(); + assertEquals("job-already-migrated", alreadyMigratedJob.getId()); + assertNull(alreadyMigratedJob.getCustomSettings()); + } + + public void testMigrateConfigs() throws InterruptedException { + + // and jobs and datafeeds clusterstate + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + mlMetadata.putJob(buildJobBuilder("job-foo").build(), false); + mlMetadata.putJob(buildJobBuilder("job-bar").build(), false); + DatafeedConfig.Builder builder = new DatafeedConfig.Builder("df-1", "job-foo"); + builder.setIndices(Collections.singletonList("beats*")); + mlMetadata.putDatafeed(builder.build(), Collections.emptyMap()); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + + ClusterService clusterService = mock(ClusterService.class); + doAnswer(invocation -> { + ClusterStateUpdateTask listener = (ClusterStateUpdateTask) invocation.getArguments()[1]; + listener.clusterStateProcessed("source", mock(ClusterState.class), mock(ClusterState.class)); + return null; + }).when(clusterService).submitStateUpdateTask(eq("remove-migrated-ml-configs"), any()); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + + // do the migration + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(client(), clusterService); + blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), + responseHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertTrue(responseHolder.get()); + + // check the jobs have been migrated + AtomicReference> jobsHolder = new AtomicReference<>(); + JobConfigProvider jobConfigProvider = new JobConfigProvider(client()); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, true, actionListener), + jobsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(jobsHolder.get(), hasSize(2)); + assertTrue(jobsHolder.get().get(0).build().getCustomSettings().containsKey(MlConfigMigrator.MIGRATED_FROM_VERSION)); + assertEquals("job-bar", jobsHolder.get().get(0).build().getId()); + assertTrue(jobsHolder.get().get(1).build().getCustomSettings().containsKey(MlConfigMigrator.MIGRATED_FROM_VERSION)); + assertEquals("job-foo", jobsHolder.get().get(1).build().getId()); + + // check datafeeds are migrated + DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client(), xContentRegistry()); + AtomicReference> datafeedsHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", true, actionListener), + datafeedsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(datafeedsHolder.get(), hasSize(1)); + assertEquals("df-1", datafeedsHolder.get().get(0).getId()); + } +} + + From 95042c0b822af31ae94c253f5a1c5ec0a2bc3549 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 7 Dec 2018 09:42:27 +0000 Subject: [PATCH 25/57] [ML] Check groups against job Ids on update (#36317) --- .../xpack/ml/job/JobManager.java | 82 +++++++++++-------- .../rest-api-spec/test/ml/jobs_crud.yml | 22 +++++ 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 3dbb5962713de..12987d3a17027 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -328,49 +328,63 @@ public void onFailure(Exception e) { public void updateJob(UpdateJobAction.Request request, ActionListener actionListener) { - ActionListener postUpdateAction; + Runnable doUpdate = () -> { + jobConfigProvider.updateJobWithValidation(request.getJobId(), request.getJobUpdate(), maxModelMemoryLimit, + this::validate, ActionListener.wrap( + updatedJob -> postJobUpdate(request, updatedJob, actionListener), + actionListener::onFailure + )); + }; - // Autodetect must be updated if the fields that the C++ uses are changed - if (request.getJobUpdate().isAutodetectProcessUpdate()) { - postUpdateAction = ActionListener.wrap( - updatedJob -> { - JobUpdate jobUpdate = request.getJobUpdate(); - if (isJobOpen(clusterService.state(), request.getJobId())) { - updateJobProcessNotifier.submitJobUpdate(UpdateParams.fromJobUpdate(jobUpdate), ActionListener.wrap( - isUpdated -> { - if (isUpdated) { - auditJobUpdatedIfNotInternal(request); - } - }, e -> { - // No need to do anything - } - )); + if (request.getJobUpdate().getGroups() != null && request.getJobUpdate().getGroups().isEmpty() == false) { + + // check the new groups are not job Ids + jobConfigProvider.jobIdMatches(request.getJobUpdate().getGroups(), ActionListener.wrap( + matchingIds -> { + if (matchingIds.isEmpty()) { + doUpdate.run(); + } else { + actionListener.onFailure(new ResourceAlreadyExistsException( + Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, matchingIds.get(0)))); } - actionListener.onResponse(new PutJobAction.Response(updatedJob)); }, actionListener::onFailure - ); + )); } else { - postUpdateAction = ActionListener.wrap(job -> { - logger.debug("[{}] No process update required for job update: {}", () -> request.getJobId(), () -> { - try { - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - request.getJobUpdate().toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); - return Strings.toString(jsonBuilder); - } catch (IOException e) { - return "(unprintable due to " + e.getMessage() + ")"; + doUpdate.run(); + } + } + + private void postJobUpdate(UpdateJobAction.Request request, Job updatedJob, ActionListener actionListener) { + // Autodetect must be updated if the fields that the C++ uses are changed + if (request.getJobUpdate().isAutodetectProcessUpdate()) { + JobUpdate jobUpdate = request.getJobUpdate(); + if (isJobOpen(clusterService.state(), request.getJobId())) { + updateJobProcessNotifier.submitJobUpdate(UpdateParams.fromJobUpdate(jobUpdate), ActionListener.wrap( + isUpdated -> { + if (isUpdated) { + auditJobUpdatedIfNotInternal(request); } - }); + }, e -> { + // No need to do anything + } + )); + } + } else { + logger.debug("[{}] No process update required for job update: {}", () -> request.getJobId(), () -> { + try { + XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); + request.getJobUpdate().toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); + return Strings.toString(jsonBuilder); + } catch (IOException e) { + return "(unprintable due to " + e.getMessage() + ")"; + } + }); - auditJobUpdatedIfNotInternal(request); - actionListener.onResponse(new PutJobAction.Response(job)); - }, - actionListener::onFailure); + auditJobUpdatedIfNotInternal(request); } - - jobConfigProvider.updateJobWithValidation(request.getJobId(), request.getJobUpdate(), maxModelMemoryLimit, - this::validate, postUpdateAction); + actionListener.onResponse(new PutJobAction.Response(updatedJob)); } private void validate(Job job, JobUpdate jobUpdate, ActionListener handler) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 285ebfc1cf9e5..f65406a25cabe 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -397,6 +397,28 @@ "description":"Can't update all description" } + - do: + xpack.ml.put_job: + job_id: job-crud-update-group-name-clash + body: > + { + "analysis_config" : { + "bucket_span": "1h", + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + + - do: + catch: "/job and group names must be unique/" + xpack.ml.update_job: + job_id: jobs-crud-update-job + body: > + { + "groups": ["job-crud-update-group-name-clash"] + } + --- "Test cannot decrease model_memory_limit below current usage": - skip: From d4b48176189f50c0dae3296e5304a233fb9413f6 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 14 Dec 2018 13:29:24 +0000 Subject: [PATCH 26/57] [ML] Adapt to periodic persistent task refresh (#36633) * [ML] Adapt to periodic persistent task refresh If https://github.com/elastic/elasticsearch/pull/36069/files is merged then the approach for reallocating ML persistent tasks after refreshing job memory requirements can be simplified. This change begins the simplification process. * Remove AwaitsFix and implement TODO --- .../xpack/core/ml/MlMetadata.java | 50 ++---------- .../xpack/ml/MachineLearning.java | 2 +- .../xpack/ml/MlConfigMigrator.java | 3 +- .../ml/action/TransportOpenJobAction.java | 11 +-- .../xpack/ml/process/MlMemoryTracker.java | 79 +++++-------------- .../xpack/ml/MlMetadataTests.java | 13 +-- .../ml/process/MlMemoryTrackerTests.java | 54 +++---------- 7 files changed, 40 insertions(+), 172 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index f0eef3c728841..76de682e99d35 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -57,9 +57,8 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom { public static final String TYPE = "ml"; private static final ParseField JOBS_FIELD = new ParseField("jobs"); private static final ParseField DATAFEEDS_FIELD = new ParseField("datafeeds"); - private static final ParseField LAST_MEMORY_REFRESH_VERSION_FIELD = new ParseField("last_memory_refresh_version"); - public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), Collections.emptySortedMap(), null); + public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), Collections.emptySortedMap()); // This parser follows the pattern that metadata is parsed leniently (to allow for enhancements) public static final ObjectParser LENIENT_PARSER = new ObjectParser<>("ml_metadata", true, Builder::new); @@ -67,18 +66,15 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom { LENIENT_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOBS_FIELD); LENIENT_PARSER.declareObjectArray(Builder::putDatafeeds, (p, c) -> DatafeedConfig.LENIENT_PARSER.apply(p, c).build(), DATAFEEDS_FIELD); - LENIENT_PARSER.declareLong(Builder::setLastMemoryRefreshVersion, LAST_MEMORY_REFRESH_VERSION_FIELD); } private final SortedMap jobs; private final SortedMap datafeeds; - private final Long lastMemoryRefreshVersion; private final GroupOrJobLookup groupOrJobLookup; - private MlMetadata(SortedMap jobs, SortedMap datafeeds, Long lastMemoryRefreshVersion) { + private MlMetadata(SortedMap jobs, SortedMap datafeeds) { this.jobs = Collections.unmodifiableSortedMap(jobs); this.datafeeds = Collections.unmodifiableSortedMap(datafeeds); - this.lastMemoryRefreshVersion = lastMemoryRefreshVersion; this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); } @@ -116,10 +112,6 @@ public Set expandDatafeedIds(String expression, boolean allowNoDatafeeds .expand(expression, allowNoDatafeeds); } - public Long getLastMemoryRefreshVersion() { - return lastMemoryRefreshVersion; - } - @Override public Version getMinimalSupportedVersion() { return Version.V_6_0_0_alpha1; @@ -153,11 +145,6 @@ public MlMetadata(StreamInput in) throws IOException { datafeeds.put(in.readString(), new DatafeedConfig(in)); } this.datafeeds = datafeeds; - if (in.getVersion().onOrAfter(Version.V_6_6_0)) { - lastMemoryRefreshVersion = in.readOptionalLong(); - } else { - lastMemoryRefreshVersion = null; - } this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); } @@ -165,9 +152,6 @@ public MlMetadata(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { writeMap(jobs, out); writeMap(datafeeds, out); - if (out.getVersion().onOrAfter(Version.V_6_6_0)) { - out.writeOptionalLong(lastMemoryRefreshVersion); - } } private static void writeMap(Map map, StreamOutput out) throws IOException { @@ -184,9 +168,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"), params); mapValuesToXContent(JOBS_FIELD, jobs, builder, extendedParams); mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, extendedParams); - if (lastMemoryRefreshVersion != null) { - builder.field(LAST_MEMORY_REFRESH_VERSION_FIELD.getPreferredName(), lastMemoryRefreshVersion); - } return builder; } @@ -203,12 +184,10 @@ public static class MlMetadataDiff implements NamedDiff { final Diff> jobs; final Diff> datafeeds; - final Long lastMemoryRefreshVersion; MlMetadataDiff(MlMetadata before, MlMetadata after) { this.jobs = DiffableUtils.diff(before.jobs, after.jobs, DiffableUtils.getStringKeySerializer()); this.datafeeds = DiffableUtils.diff(before.datafeeds, after.datafeeds, DiffableUtils.getStringKeySerializer()); - this.lastMemoryRefreshVersion = after.lastMemoryRefreshVersion; } public MlMetadataDiff(StreamInput in) throws IOException { @@ -216,11 +195,6 @@ public MlMetadataDiff(StreamInput in) throws IOException { MlMetadataDiff::readJobDiffFrom); this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new, MlMetadataDiff::readDatafeedDiffFrom); - if (in.getVersion().onOrAfter(Version.V_6_6_0)) { - lastMemoryRefreshVersion = in.readOptionalLong(); - } else { - lastMemoryRefreshVersion = null; - } } /** @@ -232,17 +206,13 @@ public MlMetadataDiff(StreamInput in) throws IOException { public MetaData.Custom apply(MetaData.Custom part) { TreeMap newJobs = new TreeMap<>(jobs.apply(((MlMetadata) part).jobs)); TreeMap newDatafeeds = new TreeMap<>(datafeeds.apply(((MlMetadata) part).datafeeds)); - // lastMemoryRefreshVersion always comes from the diff - no need to merge with the old value - return new MlMetadata(newJobs, newDatafeeds, lastMemoryRefreshVersion); + return new MlMetadata(newJobs, newDatafeeds); } @Override public void writeTo(StreamOutput out) throws IOException { jobs.writeTo(out); datafeeds.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_6_6_0)) { - out.writeOptionalLong(lastMemoryRefreshVersion); - } } @Override @@ -267,8 +237,7 @@ public boolean equals(Object o) { return false; MlMetadata that = (MlMetadata) o; return Objects.equals(jobs, that.jobs) && - Objects.equals(datafeeds, that.datafeeds) && - Objects.equals(lastMemoryRefreshVersion, that.lastMemoryRefreshVersion); + Objects.equals(datafeeds, that.datafeeds); } @Override @@ -278,14 +247,13 @@ public final String toString() { @Override public int hashCode() { - return Objects.hash(jobs, datafeeds, lastMemoryRefreshVersion); + return Objects.hash(jobs, datafeeds); } public static class Builder { private TreeMap jobs; private TreeMap datafeeds; - private Long lastMemoryRefreshVersion; public Builder() { jobs = new TreeMap<>(); @@ -299,7 +267,6 @@ public Builder(@Nullable MlMetadata previous) { } else { jobs = new TreeMap<>(previous.jobs); datafeeds = new TreeMap<>(previous.datafeeds); - lastMemoryRefreshVersion = previous.lastMemoryRefreshVersion; } } @@ -419,13 +386,8 @@ public Builder putDatafeeds(Collection datafeeds) { return this; } - public Builder setLastMemoryRefreshVersion(Long lastMemoryRefreshVersion) { - this.lastMemoryRefreshVersion = lastMemoryRefreshVersion; - return this; - } - public MlMetadata build() { - return new MlMetadata(jobs, datafeeds, lastMemoryRefreshVersion); + return new MlMetadata(jobs, datafeeds); } public void markJobAsDeleting(String jobId, PersistentTasksCustomMetaData tasks, boolean allowDeleteOpenJob) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 46cfba7e2affb..8177f813eaa7e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -419,7 +419,7 @@ public Collection createComponents(Client client, ClusterService cluster this.datafeedManager.set(datafeedManager); MlLifeCycleService mlLifeCycleService = new MlLifeCycleService(environment, clusterService, datafeedManager, autodetectProcessManager); - MlMemoryTracker memoryTracker = new MlMemoryTracker(clusterService, threadPool, jobManager, jobResultsProvider); + MlMemoryTracker memoryTracker = new MlMemoryTracker(settings, clusterService, threadPool, jobManager, jobResultsProvider); this.memoryTracker.set(memoryTracker); // This object's constructor attaches to the license state, so there's no need to retain another reference to it diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index 22dc43b9326fb..4ca5f2b839c8b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -248,8 +248,7 @@ static RemovalResult removeJobsAndDatafeeds(List jobsToRemove, List { - if (acknowledged) { - logger.trace("Job memory requirement refresh request completed successfully"); - } else { - logger.warn("Job memory requirement refresh request completed but did not set time in cluster state"); - } - }, - e -> logger.error("Failed to refresh job memory requirements", e) - )); + boolean scheduledRefresh = memoryTracker.asyncRefresh(); if (scheduledRefresh) { String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; logger.debug(reason); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java index 63f0fac27d8e8..7b689376c9e79 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java @@ -9,19 +9,15 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.LocalNodeMasterListener; -import org.elasticsearch.cluster.ack.AckedRequest; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; @@ -44,22 +40,10 @@ * 1. For all open ML jobs (via {@link #asyncRefresh}) * 2. For all open ML jobs, plus one named ML job that is not open (via {@link #refreshJobMemoryAndAllOthers}) * 3. For one named ML job (via {@link #refreshJobMemory}) - * In all cases a listener informs the caller when the requested updates are complete. + * In cases 2 and 3 a listener informs the caller when the requested updates are complete. */ public class MlMemoryTracker implements LocalNodeMasterListener { - private static final AckedRequest ACKED_REQUEST = new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; - } - - @Override - public TimeValue masterNodeTimeout() { - return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; - } - }; - private static final Duration RECENT_UPDATE_THRESHOLD = Duration.ofMinutes(1); private final Logger logger = LogManager.getLogger(MlMemoryTracker.class); @@ -72,14 +56,22 @@ public TimeValue masterNodeTimeout() { private final JobResultsProvider jobResultsProvider; private volatile boolean isMaster; private volatile Instant lastUpdateTime; + private volatile Duration reassignmentRecheckInterval; - public MlMemoryTracker(ClusterService clusterService, ThreadPool threadPool, JobManager jobManager, + public MlMemoryTracker(Settings settings, ClusterService clusterService, ThreadPool threadPool, JobManager jobManager, JobResultsProvider jobResultsProvider) { this.threadPool = threadPool; this.clusterService = clusterService; this.jobManager = jobManager; this.jobResultsProvider = jobResultsProvider; + setReassignmentRecheckInterval(PersistentTasksClusterService.CLUSTER_TASKS_ALLOCATION_RECHECK_INTERVAL_SETTING.get(settings)); clusterService.addLocalNodeMasterListener(this); + clusterService.getClusterSettings().addSettingsUpdateConsumer( + PersistentTasksClusterService.CLUSTER_TASKS_ALLOCATION_RECHECK_INTERVAL_SETTING, this::setReassignmentRecheckInterval); + } + + private void setReassignmentRecheckInterval(TimeValue recheckInterval) { + reassignmentRecheckInterval = Duration.ofNanos(recheckInterval.getNanos()); } @Override @@ -103,11 +95,12 @@ public String executorName() { /** * Is the information in this object sufficiently up to date - * for valid allocation decisions to be made using it? + * for valid task assignment decisions to be made using it? */ public boolean isRecentlyRefreshed() { Instant localLastUpdateTime = lastUpdateTime; - return localLastUpdateTime != null && localLastUpdateTime.plus(RECENT_UPDATE_THRESHOLD).isAfter(Instant.now()); + return localLastUpdateTime != null && + localLastUpdateTime.plus(RECENT_UPDATE_THRESHOLD).plus(reassignmentRecheckInterval).isAfter(Instant.now()); } /** @@ -143,24 +136,19 @@ public void removeJob(String jobId) { /** * Uses a separate thread to refresh the memory requirement for every ML job that has * a corresponding persistent task. This method only works on the master node. - * @param listener Will be called when the async refresh completes or fails. The - * boolean value indicates whether the cluster state was updated - * with the refresh completion time. (If it was then this will in - * cause the persistent tasks framework to check if any persistent - * tasks are awaiting allocation.) * @return true if the async refresh is scheduled, and false * if this is not possible for some reason. */ - public boolean asyncRefresh(ActionListener listener) { + public boolean asyncRefresh() { if (isMaster) { try { - ActionListener mlMetaUpdateListener = ActionListener.wrap( - aVoid -> recordUpdateTimeInClusterState(listener), - listener::onFailure + ActionListener listener = ActionListener.wrap( + aVoid -> logger.trace("Job memory requirement refresh request completed successfully"), + e -> logger.error("Failed to refresh job memory requirements", e) ); threadPool.executor(executorName()).execute( - () -> refresh(clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE), mlMetaUpdateListener)); + () -> refresh(clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE), listener)); return true; } catch (EsRejectedExecutionException e) { logger.debug("Couldn't schedule ML memory update - node might be shutting down", e); @@ -227,33 +215,6 @@ void refresh(PersistentTasksCustomMetaData persistentTasks, ActionListener } } - private void recordUpdateTimeInClusterState(ActionListener listener) { - - clusterService.submitStateUpdateTask("ml-memory-last-update-time", - new AckedClusterStateUpdateTask(ACKED_REQUEST, listener) { - @Override - protected Boolean newResponse(boolean acknowledged) { - return acknowledged; - } - - @Override - public ClusterState execute(ClusterState currentState) { - MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(currentState); - MlMetadata.Builder builder = new MlMetadata.Builder(currentMlMetadata); - builder.setLastMemoryRefreshVersion(currentState.getVersion() + 1); - MlMetadata newMlMetadata = builder.build(); - if (newMlMetadata.equals(currentMlMetadata)) { - // Return same reference if nothing has changed - return currentState; - } else { - ClusterState.Builder newState = ClusterState.builder(currentState); - newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMlMetadata).build()); - return newState.build(); - } - } - }); - } - private void iterateMlJobTasks(Iterator> iterator, ActionListener refreshComplete) { if (iterator.hasNext()) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java index eb58221bf5f35..c7ca2ff805eba 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java @@ -69,9 +69,6 @@ protected MlMetadata createTestInstance() { builder.putJob(job, false); } } - if (randomBoolean()) { - builder.setLastMemoryRefreshVersion(randomNonNegativeLong()); - } return builder.build(); } @@ -441,9 +438,8 @@ protected MlMetadata mutateInstance(MlMetadata instance) { for (Map.Entry entry : datafeeds.entrySet()) { metadataBuilder.putDatafeed(entry.getValue(), Collections.emptyMap()); } - metadataBuilder.setLastMemoryRefreshVersion(instance.getLastMemoryRefreshVersion()); - switch (between(0, 2)) { + switch (between(0, 1)) { case 0: metadataBuilder.putJob(JobTests.createRandomizedJob(), true); break; @@ -463,13 +459,6 @@ protected MlMetadata mutateInstance(MlMetadata instance) { metadataBuilder.putJob(randomJob, false); metadataBuilder.putDatafeed(datafeedConfig, Collections.emptyMap()); break; - case 2: - if (instance.getLastMemoryRefreshVersion() == null) { - metadataBuilder.setLastMemoryRefreshVersion(randomNonNegativeLong()); - } else { - metadataBuilder.setLastMemoryRefreshVersion(null); - } - break; default: throw new AssertionError("Illegal randomisation branch"); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java index cbba7ffa04972..197fa469bed7c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java @@ -6,14 +6,14 @@ package org.elasticsearch.xpack.ml.process; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; @@ -22,14 +22,13 @@ import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.junit.Before; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.anyString; @@ -42,8 +41,6 @@ public class MlMemoryTrackerTests extends ESTestCase { - private ClusterService clusterService; - private ThreadPool threadPool; private JobManager jobManager; private JobResultsProvider jobResultsProvider; private MlMemoryTracker memoryTracker; @@ -51,8 +48,11 @@ public class MlMemoryTrackerTests extends ESTestCase { @Before public void setup() { - clusterService = mock(ClusterService.class); - threadPool = mock(ThreadPool.class); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, + Collections.singleton(PersistentTasksClusterService.CLUSTER_TASKS_ALLOCATION_RECHECK_INTERVAL_SETTING)); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + ThreadPool threadPool = mock(ThreadPool.class); ExecutorService executorService = mock(ExecutorService.class); doAnswer(invocation -> { @SuppressWarnings("unchecked") @@ -63,7 +63,7 @@ public void setup() { when(threadPool.executor(anyString())).thenReturn(executorService); jobManager = mock(JobManager.class); jobResultsProvider = mock(JobResultsProvider.class); - memoryTracker = new MlMemoryTracker(clusterService, threadPool, jobManager, jobResultsProvider); + memoryTracker = new MlMemoryTracker(Settings.EMPTY, clusterService, threadPool, jobManager, jobResultsProvider); } public void testRefreshAll() { @@ -154,40 +154,6 @@ public void testRefreshOne() { assertNull(memoryTracker.getJobMemoryRequirement(jobId)); } - @SuppressWarnings("unchecked") - public void testRecordUpdateTimeInClusterState() { - - boolean isMaster = randomBoolean(); - if (isMaster) { - memoryTracker.onMaster(); - } else { - memoryTracker.offMaster(); - } - - when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); - - AtomicReference updateVersion = new AtomicReference<>(); - - doAnswer(invocation -> { - AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocation.getArguments()[1]; - ClusterState currentClusterState = ClusterState.EMPTY_STATE; - ClusterState newClusterState = task.execute(currentClusterState); - assertThat(currentClusterState, not(equalTo(newClusterState))); - MlMetadata newMlMetadata = MlMetadata.getMlMetadata(newClusterState); - updateVersion.set(newMlMetadata.getLastMemoryRefreshVersion()); - task.onAllNodesAcked(null); - return null; - }).when(clusterService).submitStateUpdateTask(anyString(), any(AckedClusterStateUpdateTask.class)); - - memoryTracker.asyncRefresh(ActionListener.wrap(ESTestCase::assertTrue, ESTestCase::assertNull)); - - if (isMaster) { - assertNotNull(updateVersion.get()); - } else { - assertNull(updateVersion.get()); - } - } - private PersistentTasksCustomMetaData.PersistentTask makeTestTask(String jobId) { return new PersistentTasksCustomMetaData.PersistentTask<>("job-" + jobId, MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId), 0, PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT); From 9bb881f5ac9d24ced7d7f4b5570f4a97fc02c3f6 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 12 Dec 2018 12:14:41 +0000 Subject: [PATCH 27/57] [ML] Default search size for configs --- .../persistence/DatafeedConfigProvider.java | 11 +++++++++- .../ml/job/persistence/JobConfigProvider.java | 22 ++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index 8e04d991036fe..5367278be9ac2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -87,6 +87,13 @@ public class DatafeedConfigProvider { TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable); } + /** + * In most cases we expect 10s or 100s of datafeeds to be defined and + * a search for all datafeeds should return all. + * TODO this is a temporary fix + */ + public int searchSize = 1000; + public DatafeedConfigProvider(Client client, NamedXContentRegistry xContentRegistry) { this.client = client; this.xContentRegistry = xContentRegistry; @@ -408,7 +415,9 @@ public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, A SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setSource(sourceBuilder).request(); + .setSource(sourceBuilder) + .setSize(searchSize) + .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 0fe93c9e40975..04f09a39247f5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -101,6 +101,13 @@ public class JobConfigProvider { TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable); } + /** + * In most cases we expect 10s or 100s of jobs to be defined and + * a search for all jobs should return all. + * TODO this is a temporary fix + */ + private int searchSize = 1000; + private final Client client; public JobConfigProvider(Client client) { @@ -610,7 +617,9 @@ public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDe SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setSource(sourceBuilder).request(); + .setSource(sourceBuilder) + .setSize(searchSize) + .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); @@ -666,7 +675,9 @@ public void expandGroupIds(List groupIds, ActionListenerwrap( @@ -722,12 +733,13 @@ public void findJobsWithCustomRules(ActionListener> listener) { String customRulesPath = Strings.collectionToDelimitedString(Arrays.asList(Job.ANALYSIS_CONFIG.getPreferredName(), AnalysisConfig.DETECTORS.getPreferredName(), Detector.CUSTOM_RULES_FIELD.getPreferredName()), "."); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .query(QueryBuilders.nestedQuery(customRulesPath, QueryBuilders.existsQuery(customRulesPath), ScoreMode.None)) - .size(10000); + .query(QueryBuilders.nestedQuery(customRulesPath, QueryBuilders.existsQuery(customRulesPath), ScoreMode.None)); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setSource(sourceBuilder).request(); + .setSource(sourceBuilder) + .setSize(searchSize) + .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( From cee07b1504d6e423c5f7605d15c78847b1add983 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 14 Dec 2018 16:57:45 +0000 Subject: [PATCH 28/57] Fix TooManyJobsIT.testMultipleNodes Two problems: 1. Stack overflow during async iteration when lots of jobs on same machine 2. Not effectively setting search size in all cases --- .../xpack/ml/job/persistence/JobConfigProvider.java | 7 +++++-- .../elasticsearch/xpack/ml/process/MlMemoryTracker.java | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 04f09a39247f5..49f82bba0335d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -478,8 +478,9 @@ public void jobIdMatches(List ids, ActionListener> listener SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(sourceBuilder) .setSize(ids.size()) - .setSource(sourceBuilder).request(); + .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, ActionListener.wrap( @@ -563,7 +564,9 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setSource(sourceBuilder).request(); + .setSource(sourceBuilder) + .setSize(searchSize) + .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java index 7b689376c9e79..b60e4dd4a2e9d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java @@ -220,7 +220,13 @@ private void iterateMlJobTasks(Iterator iterateMlJobTasks(iterator, refreshComplete), refreshComplete::onFailure)); + ActionListener.wrap( + // Do the next iteration in a different thread, otherwise stack overflow + // can occur if the searches happen to be on the local node, as the huge + // chain of listeners are all called in the same thread if only one node + // is involved + mem -> threadPool.executor(executorName()).submit(() -> iterateMlJobTasks(iterator, refreshComplete)), + refreshComplete::onFailure)); } else { refreshComplete.onResponse(null); } From 95df75190d5145b87ded1a3ec099a4af06d8c1ca Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 14 Dec 2018 18:01:38 +0000 Subject: [PATCH 29/57] Use execute() instead of submit() in MlMemoryTracker We don't need a Future to wait for completion --- .../org/elasticsearch/xpack/ml/process/MlMemoryTracker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java index b60e4dd4a2e9d..54a7400375fe9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java @@ -225,7 +225,7 @@ private void iterateMlJobTasks(Iterator threadPool.executor(executorName()).submit(() -> iterateMlJobTasks(iterator, refreshComplete)), + mem -> threadPool.executor(executorName()).execute(() -> iterateMlJobTasks(iterator, refreshComplete)), refreshComplete::onFailure)); } else { refreshComplete.onResponse(null); From cbe9099409c658e1a90383befdb13716221efd1f Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 12 Dec 2018 18:27:17 +0000 Subject: [PATCH 30/57] [ML][TEST] Fix NPE in JobManagerTests --- .../xpack/ml/job/persistence/MockClientBuilder.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 96ce4fb388f2d..627465f1d4f21 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -283,6 +283,7 @@ public MockClientBuilder prepareSearch(String indexName, List do when(builder.setIndicesOptions(any())).thenReturn(builder); when(builder.setQuery(any())).thenReturn(builder); when(builder.setSource(any())).thenReturn(builder); + when(builder.setSize(anyInt())).thenReturn(builder); SearchRequest request = new SearchRequest(indexName); when(builder.request()).thenReturn(request); @@ -321,6 +322,7 @@ public MockClientBuilder prepareSearchFields(String indexName, List Date: Sun, 16 Dec 2018 09:42:51 +0000 Subject: [PATCH 31/57] [ML] JIindex: Limit the size of bulk migrations (#36481) --- .../xpack/ml/MlConfigMigrator.java | 83 +++++++++++++++++-- .../xpack/ml/MlConfigMigratorTests.java | 80 ++++++++++++++++++ 2 files changed, 155 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index 4ca5f2b839c8b..75bf4bec25b3a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -38,11 +38,13 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -68,7 +70,10 @@ * If there was an error in step 3 and the config is in both the clusterstate and * index then when the migrator retries it must not overwrite an existing job config * document as once the index document is present all update operations will function - * on that rather than the clusterstate + * on that rather than the clusterstate. + * + * The number of configs indexed in each bulk operation is limited by {@link #MAX_BULK_WRITE_SIZE} + * pairs of datafeeds and jobs are migrated together. */ public class MlConfigMigrator { @@ -76,6 +81,8 @@ public class MlConfigMigrator { public static final String MIGRATED_FROM_VERSION = "migrated from version"; + static final int MAX_BULK_WRITE_SIZE = 100; + private final Client client; private final ClusterService clusterService; @@ -111,10 +118,12 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener return; } - Collection datafeedsToMigrate = stoppedDatafeedConfigs(clusterState); - List jobsToMigrate = nonDeletingJobs(closedJobConfigs(clusterState)).stream() + Collection stoppedDatafeeds = stoppedDatafeedConfigs(clusterState); + Map eligibleJobs = nonDeletingJobs(closedJobConfigs(clusterState)).stream() .map(MlConfigMigrator::updateJobForMigration) - .collect(Collectors.toList()); + .collect(Collectors.toMap(Job::getId, Function.identity(), (a, b) -> a)); + + JobsAndDatafeeds jobsAndDatafeedsToMigrate = limitWrites(stoppedDatafeeds, eligibleJobs); ActionListener unMarkMigrationInProgress = ActionListener.wrap( response -> { @@ -127,16 +136,18 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener } ); - if (datafeedsToMigrate.isEmpty() && jobsToMigrate.isEmpty()) { + if (jobsAndDatafeedsToMigrate.totalCount() == 0) { unMarkMigrationInProgress.onResponse(Boolean.FALSE); return; } - writeConfigToIndex(datafeedsToMigrate, jobsToMigrate, ActionListener.wrap( + logger.debug("migrating ml configurations"); + + writeConfigToIndex(jobsAndDatafeedsToMigrate.datafeedConfigs, jobsAndDatafeedsToMigrate.jobs, ActionListener.wrap( failedDocumentIds -> { - List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, jobsToMigrate); + List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.jobs); List successfulDatafeedWrites = - filterFailedDatafeedConfigWrites(failedDocumentIds, datafeedsToMigrate); + filterFailedDatafeedConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.datafeedConfigs); removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, unMarkMigrationInProgress); }, unMarkMigrationInProgress::onFailure @@ -341,6 +352,62 @@ public static List stoppedDatafeedConfigs(ClusterState clusterSt .collect(Collectors.toList()); } + public static class JobsAndDatafeeds { + List jobs; + List datafeedConfigs; + + private JobsAndDatafeeds() { + jobs = new ArrayList<>(); + datafeedConfigs = new ArrayList<>(); + } + + public int totalCount() { + return jobs.size() + datafeedConfigs.size(); + } + } + + /** + * Return at most {@link #MAX_BULK_WRITE_SIZE} configs favouring + * datafeed and job pairs so if a datafeed is chosen so is its job. + * + * @param datafeedsToMigrate Datafeed configs + * @param jobsToMigrate Job configs + * @return Job and datafeed configs + */ + public static JobsAndDatafeeds limitWrites(Collection datafeedsToMigrate, Map jobsToMigrate) { + JobsAndDatafeeds jobsAndDatafeeds = new JobsAndDatafeeds(); + + if (datafeedsToMigrate.size() + jobsToMigrate.size() <= MAX_BULK_WRITE_SIZE) { + jobsAndDatafeeds.jobs.addAll(jobsToMigrate.values()); + jobsAndDatafeeds.datafeedConfigs.addAll(datafeedsToMigrate); + return jobsAndDatafeeds; + } + + int count = 0; + + // prioritise datafeed and job pairs + for (DatafeedConfig datafeedConfig : datafeedsToMigrate) { + if (count < MAX_BULK_WRITE_SIZE) { + jobsAndDatafeeds.datafeedConfigs.add(datafeedConfig); + count++; + Job datafeedsJob = jobsToMigrate.remove(datafeedConfig.getJobId()); + if (datafeedsJob != null) { + jobsAndDatafeeds.jobs.add(datafeedsJob); + count++; + } + } + } + + // are there jobs without datafeeds to migrate + Iterator iter = jobsToMigrate.values().iterator(); + while (iter.hasNext() && count < MAX_BULK_WRITE_SIZE) { + jobsAndDatafeeds.jobs.add(iter.next()); + count++; + } + + return jobsAndDatafeeds; + } + /** * Check for failures in the bulk response and return the * Ids of any documents not written to the index diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java index faaf4425dfb02..2fe85c4d1a057 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java @@ -24,8 +24,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -212,6 +215,83 @@ public void testRemoveJobsAndDatafeeds_removeSome() { } + public void testLimitWrites_GivenBelowLimit() { + MlConfigMigrator.JobsAndDatafeeds jobsAndDatafeeds = MlConfigMigrator.limitWrites(Collections.emptyList(), Collections.emptyMap()); + assertThat(jobsAndDatafeeds.datafeedConfigs, empty()); + assertThat(jobsAndDatafeeds.jobs, empty()); + + List datafeeds = new ArrayList<>(); + Map jobs = new HashMap<>(); + + int numDatafeeds = MlConfigMigrator.MAX_BULK_WRITE_SIZE / 2; + for (int i=0; i datafeeds = new ArrayList<>(); + Map jobs = new HashMap<>(); + + int numDatafeeds = MlConfigMigrator.MAX_BULK_WRITE_SIZE / 2 + 10; + for (int i=0; i selectedJobIds = jobsAndDatafeeds.jobs.stream().map(Job::getId).collect(Collectors.toSet()); + Set datafeedJobIds = jobsAndDatafeeds.datafeedConfigs.stream().map(DatafeedConfig::getJobId).collect(Collectors.toSet()); + assertEquals(selectedJobIds, datafeedJobIds); + } + + public void testLimitWrites_GivenMoreJobsThanDatafeeds() { + List datafeeds = new ArrayList<>(); + Map jobs = new HashMap<>(); + + int numDatafeeds = MlConfigMigrator.MAX_BULK_WRITE_SIZE / 2 - 10; + for (int i=0; i selectedJobIds = jobsAndDatafeeds.jobs.stream().map(Job::getId).collect(Collectors.toSet()); + Set datafeedJobIds = jobsAndDatafeeds.datafeedConfigs.stream().map(DatafeedConfig::getJobId).collect(Collectors.toSet()); + assertTrue(selectedJobIds.containsAll(datafeedJobIds)); + } + + public void testLimitWrites_GivenNullJob() { + List datafeeds = Collections.singletonList(createCompatibleDatafeed("no-job-for-this-datafeed")); + MlConfigMigrator.JobsAndDatafeeds jobsAndDatafeeds = MlConfigMigrator.limitWrites(datafeeds, Collections.emptyMap()); + + assertThat(jobsAndDatafeeds.datafeedConfigs, hasSize(1)); + assertThat(jobsAndDatafeeds.jobs, empty()); + } + private DatafeedConfig createCompatibleDatafeed(String jobId) { // create a datafeed without aggregations or anything // else that may cause validation errors From 5eb20d21d72de101b713f4329065a75440312e1a Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 17 Dec 2018 13:56:03 +0000 Subject: [PATCH 32/57] [ML] Prevent updates and upgrade tests (#36649) --- .../xpack/core/ml/utils/ExceptionsHelper.java | 5 + .../xpack/test/rest/XPackRestTestHelper.java | 27 ++- .../xpack/ml/MlAssignmentNotifier.java | 29 +-- .../xpack/ml/MlConfigMigrator.java | 63 ++++++ .../action/TransportDeleteDatafeedAction.java | 11 +- .../ml/action/TransportDeleteJobAction.java | 7 + .../ml/action/TransportOpenJobAction.java | 6 + .../TransportRevertModelSnapshotAction.java | 6 + .../action/TransportStartDatafeedAction.java | 6 + .../action/TransportUpdateDatafeedAction.java | 7 + .../xpack/ml/job/JobManager.java | 7 + .../xpack/ml/MlAssignmentNotifierTests.java | 34 --- .../xpack/ml/MlConfigMigratorTests.java | 121 ++++++++++ .../xpack/ml/job/JobManagerTests.java | 22 +- .../xpack/restart/FullClusterRestartIT.java | 7 - .../MlMigrationFullClusterRestartIT.java | 211 ++++++++++++++++++ .../UpgradeClusterClientYamlTestSuiteIT.java | 3 +- 17 files changed, 502 insertions(+), 70 deletions(-) create mode 100644 x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java index 83bbe79a7b470..47c0d4f64f96f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java @@ -58,6 +58,11 @@ public static ElasticsearchStatusException badRequestException(String msg, Objec return new ElasticsearchStatusException(msg, RestStatus.BAD_REQUEST, args); } + public static ElasticsearchStatusException configHasNotBeenMigrated(String verb, String id) { + return new ElasticsearchStatusException("cannot {} as the configuration [{}] is temporarily pending migration", + RestStatus.SERVICE_UNAVAILABLE, verb, id); + } + /** * Creates an error message that explains there are shard failures, displays info * for the first failure (shard/reason) and kindly asks to see more info in the logs diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java index 28e96d0974ea5..47580bf731a44 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java @@ -20,20 +20,38 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public final class XPackRestTestHelper { + public static final List ML_PRE_V660_TEMPLATES = Collections.unmodifiableList( + Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, + MlMetaIndex.INDEX_NAME, + AnomalyDetectorsIndex.jobStateIndexName(), + AnomalyDetectorsIndex.jobResultsIndexPrefix())); + + public static final List ML_POST_V660_TEMPLATES = Collections.unmodifiableList( + Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, + MlMetaIndex.INDEX_NAME, + AnomalyDetectorsIndex.jobStateIndexName(), + AnomalyDetectorsIndex.jobResultsIndexPrefix(), + AnomalyDetectorsIndex.configIndexName())); + private XPackRestTestHelper() { } /** - * Waits for the Machine Learning templates to be created - * and check the version is up to date + * For each template name wait for the template to be created and + * for the template version to be equal to the master node version. + * + * @param client The rest client + * @param templateNames Names of the templates to wait for + * @throws InterruptedException If the wait is interrupted */ - public static void waitForMlTemplates(RestClient client) throws InterruptedException { + public static void waitForTemplates(RestClient client, List templateNames) throws InterruptedException { AtomicReference masterNodeVersion = new AtomicReference<>(); ESTestCase.awaitBusy(() -> { String response; @@ -53,8 +71,6 @@ public static void waitForMlTemplates(RestClient client) throws InterruptedExcep return false; }); - final List templateNames = Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, - AnomalyDetectorsIndex.jobStateIndexName(), AnomalyDetectorsIndex.jobResultsIndexPrefix()); for (String template : templateNames) { ESTestCase.awaitBusy(() -> { Map response; @@ -74,5 +90,4 @@ public static void waitForMlTemplates(RestClient client) throws InterruptedExcep }); } } - } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index 4e6db3fa0bbff..ead55aa10cbd4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -7,7 +7,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -83,29 +82,23 @@ public void clusterChanged(ClusterChangedEvent event) { } PersistentTasksCustomMetaData previous = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData current = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - if (Objects.equals(previous, current)) { - return; - } - - Version minNodeVersion = event.state().nodes().getMinNodeVersion(); - if (minNodeVersion.onOrAfter(Version.V_6_6_0)) { - // ok to migrate - mlConfigMigrator.migrateConfigsWithoutTasks(event.state(), ActionListener.wrap( - response -> threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())), - e -> { - logger.error("error migrating ml configurations", e); - threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); - } - )); - } else { - threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); - } + mlConfigMigrator.migrateConfigsWithoutTasks(event.state(), ActionListener.wrap( + response -> threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())), + e -> { + logger.error("error migrating ml configurations", e); + threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); + } + )); } private void auditChangesToMlTasks(PersistentTasksCustomMetaData current, PersistentTasksCustomMetaData previous, ClusterState state) { + if (Objects.equals(previous, current)) { + return; + } + for (PersistentTask currentTask : current.tasks()) { Assignment currentAssignment = currentTask.getAssignment(); PersistentTask previousTask = previous != null ? previous.getTask(currentTask.getId()) : null; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index 75bf4bec25b3a..2490b820440bb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -80,6 +80,7 @@ public class MlConfigMigrator { private static final Logger logger = LogManager.getLogger(MlConfigMigrator.class); public static final String MIGRATED_FROM_VERSION = "migrated from version"; + public static final Version MIN_NODE_VERSION = Version.V_6_6_0; static final int MAX_BULK_WRITE_SIZE = 100; @@ -113,6 +114,12 @@ public MlConfigMigrator(Client client, ClusterService clusterService) { */ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener listener) { + Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); + if (minNodeVersion.before(MIN_NODE_VERSION)) { + listener.onResponse(Boolean.FALSE); + return; + } + if (migrationInProgress.compareAndSet(false, true) == false) { listener.onResponse(Boolean.FALSE); return; @@ -447,4 +454,60 @@ static List filterFailedDatafeedConfigWrites(Set failedDocumentI .filter(id -> failedDocumentIds.contains(DatafeedConfig.documentId(id)) == false) .collect(Collectors.toList()); } + + /** + * Is the job a eligible for migration? Returns: + * False if the min node version of the cluster is before {@link #MIN_NODE_VERSION} + * False if the job is not in the cluster state + * False if the {@link Job#isDeleting()} + * False if the job has a persistent task + * True otherwise i.e. the job is present, not deleting + * and does not have a persistent task. + * + * @param jobId The job Id + * @param clusterState clusterstate + * @return A boolean depending on the conditions listed above + */ + public static boolean jobIsEligibleForMigration(String jobId, ClusterState clusterState) { + Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); + if (minNodeVersion.before(MIN_NODE_VERSION)) { + return false; + } + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + Job job = mlMetadata.getJobs().get(jobId); + + if (job == null || job.isDeleting()) { + return false; + } + + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + return MlTasks.openJobIds(persistentTasks).contains(jobId) == false; + } + + /** + * Is the datafeed a eligible for migration? Returns: + * False if the min node version of the cluster is before {@link #MIN_NODE_VERSION} + * False if the datafeed is not in the cluster state + * False if the datafeed has a persistent task + * True otherwise i.e. the datafeed is present and does not have a persistent task. + * + * @param datafeedId The datafeed Id + * @param clusterState clusterstate + * @return A boolean depending on the conditions listed above + */ + public static boolean datafeedIsEligibleForMigration(String datafeedId, ClusterState clusterState) { + Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); + if (minNodeVersion.before(MIN_NODE_VERSION)) { + return false; + } + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + if (mlMetadata.getDatafeeds().containsKey(datafeedId) == false) { + return false; + } + + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + return MlTasks.startedDatafeedIds(persistentTasks).contains(datafeedId) == false; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java index b541aff474e63..42be2c7b9ddcd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java @@ -17,9 +17,9 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksService; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -66,7 +67,13 @@ protected AcknowledgedResponse newResponse() { @Override protected void masterOperation(DeleteDatafeedAction.Request request, ClusterState state, - ActionListener listener) throws Exception { + ActionListener listener) { + + if (MlConfigMigrator.datafeedIsEligibleForMigration(request.getDatafeedId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("delete datafeed", request.getDatafeedId())); + return; + } + if (request.isForce()) { forceDeleteDatafeed(request, state, listener); } else { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 67d79431c28e6..1c4505e93afbb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -63,6 +63,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; @@ -146,6 +147,12 @@ protected void masterOperation(DeleteJobAction.Request request, ClusterState sta @Override protected void masterOperation(Task task, DeleteJobAction.Request request, ClusterState state, ActionListener listener) { + + if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("delete job", request.getJobId())); + return; + } + logger.debug("Deleting job '{}'", request.getJobId()); if (request.isForce() == false) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 95b6bd37ffee4..433d1b420303f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -67,6 +67,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -503,6 +504,11 @@ protected ClusterBlockException checkBlock(OpenJobAction.Request request, Cluste @Override protected void masterOperation(OpenJobAction.Request request, ClusterState state, ActionListener listener) { + if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobParams().getJobId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("open job", request.getJobParams().getJobId())); + return; + } + OpenJobAction.JobParams jobParams = request.getJobParams(); if (licenseState.isMachineLearningAllowed()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index 34fe32921c062..cf15f6ab41a66 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; @@ -67,6 +68,11 @@ protected RevertModelSnapshotAction.Response newResponse() { @Override protected void masterOperation(RevertModelSnapshotAction.Request request, ClusterState state, ActionListener listener) { + if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("revert model snapshot", request.getJobId())); + return; + } + logger.debug("Received request to revert to snapshot id '{}' for job '{}', deleting intervening results: {}", request.getSnapshotId(), request.getJobId(), request.getDeleteInterveningResults()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index 59fb281abe371..9fa0f5a5acb87 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.DatafeedNodeSelector; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; @@ -137,6 +138,11 @@ protected void masterOperation(StartDatafeedAction.Request request, ClusterState return; } + if (MlConfigMigrator.datafeedIsEligibleForMigration(request.getParams().getDatafeedId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("start datafeed", request.getParams().getDatafeedId())); + return; + } + AtomicReference datafeedConfigHolder = new AtomicReference<>(); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java index 7bf8398dc2e5a..3f89bbd70853a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; @@ -62,6 +63,12 @@ protected PutDatafeedAction.Response newResponse() { @Override protected void masterOperation(UpdateDatafeedAction.Request request, ClusterState state, ActionListener listener) throws Exception { + + if (MlConfigMigrator.datafeedIsEligibleForMigration(request.getUpdate().getId(), state)) { + listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("update datafeed", request.getUpdate().getId())); + return; + } + final Map headers = threadPool.getThreadContext().getHeaders(); // Check datafeed is stopped diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 12987d3a17027..291347028453b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -48,6 +48,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; @@ -336,6 +337,12 @@ public void updateJob(UpdateJobAction.Request request, ActionListener fail("response not expected: " + response), + exception -> { + assertThat(exception, instanceOf(ElasticsearchStatusException.class)); + } + )); + + } + + public void testUpdateProcessOnCalendarChanged() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job-1", "node_id", JobState.OPENED, tasksBuilder); addJobTask("job-2", "node_id", JobState.OPENED, tasksBuilder); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 595edb76b7cb0..beea8c9a21856 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -23,14 +23,12 @@ import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; import org.elasticsearch.xpack.security.support.SecurityIndexManager; -import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.hamcrest.Matcher; -import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -59,11 +57,6 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { - @Before - public void waitForMlTemplates() throws Exception { - XPackRestTestHelper.waitForMlTemplates(client()); - } - @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java new file mode 100644 index 0000000000000..6a6617f267f73 --- /dev/null +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.restart; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.DataDescription; +import org.elasticsearch.xpack.core.ml.job.config.Detector; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.isEmptyOrNullString; + +public class MlMigrationFullClusterRestartIT extends AbstractFullClusterRestartTestCase { + + private static final String OLD_CLUSTER_CLOSED_JOB_ID = "migration-old-cluster-closed-job"; + private static final String OLD_CLUSTER_STOPPED_DATAFEED_ID = "migration-old-cluster-stopped-datafeed"; + + @Override + protected Settings restClientSettings() { + String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + + @Before + public void waitForMlTemplates() throws Exception { + List templatesToWaitFor = XPackRestTestHelper.ML_POST_V660_TEMPLATES; + + // If upgrading from a version prior to v6.6.0 the set of templates + // to wait for is different + if (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_6_6_0) ) { + templatesToWaitFor = XPackRestTestHelper.ML_PRE_V660_TEMPLATES; + } + + XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); + } + + private void createTestIndex() throws IOException { + Request createTestIndex = new Request("PUT", "/airline-data"); + createTestIndex.setJsonEntity("{\"mappings\": { \"doc\": {\"properties\": {" + + "\"time\": {\"type\": \"date\"}," + + "\"airline\": {\"type\": \"keyword\"}," + + "\"responsetime\": {\"type\": \"float\"}" + + "}}}}"); + client().performRequest(createTestIndex); + } + + public void testMigration() throws Exception { + if (isRunningAgainstOldCluster()) { + createTestIndex(); + oldClusterTests(); + } else { + upgradedClusterTests(); + } + } + + private void oldClusterTests() throws IOException { + // create jobs and datafeeds + Detector.Builder d = new Detector.Builder("metric", "responsetime"); + d.setByFieldName("airline"); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + analysisConfig.setBucketSpan(TimeValue.timeValueMinutes(10)); + + Job.Builder closedJob = new Job.Builder(OLD_CLUSTER_CLOSED_JOB_ID); + closedJob.setAnalysisConfig(analysisConfig); + closedJob.setDataDescription(new DataDescription.Builder()); + + Request putClosedJob = new Request("PUT", "/_xpack/ml/anomaly_detectors/" + OLD_CLUSTER_CLOSED_JOB_ID); + putClosedJob.setJsonEntity(Strings.toString(closedJob)); + client().performRequest(putClosedJob); + + DatafeedConfig.Builder stoppedDfBuilder = new DatafeedConfig.Builder(OLD_CLUSTER_STOPPED_DATAFEED_ID, OLD_CLUSTER_CLOSED_JOB_ID); + if (getOldClusterVersion().before(Version.V_6_6_0)) { + stoppedDfBuilder.setDelayedDataCheckConfig(null); + } + stoppedDfBuilder.setIndices(Collections.singletonList("airline-data")); + + Request putStoppedDatafeed = new Request("PUT", "/_xpack/ml/datafeeds/" + OLD_CLUSTER_STOPPED_DATAFEED_ID); + putStoppedDatafeed.setJsonEntity(Strings.toString(stoppedDfBuilder.build())); + client().performRequest(putStoppedDatafeed); + } + + private void upgradedClusterTests() throws Exception { + // wait for the closed job and datafeed to be migrated + waitForMigration(Collections.singletonList(OLD_CLUSTER_CLOSED_JOB_ID), + Collections.singletonList(OLD_CLUSTER_STOPPED_DATAFEED_ID), + Collections.emptyList(), Collections.emptyList()); + + // open the migrated job and datafeed + Request openJob = new Request("POST", "_ml/anomaly_detectors/" + OLD_CLUSTER_CLOSED_JOB_ID + "/_open"); + client().performRequest(openJob); + Request startDatafeed = new Request("POST", "_ml/datafeeds/" + OLD_CLUSTER_STOPPED_DATAFEED_ID + "/_start"); + client().performRequest(startDatafeed); + + waitForJobToBeAssigned(OLD_CLUSTER_CLOSED_JOB_ID); + waitForDatafeedToBeAssigned(OLD_CLUSTER_STOPPED_DATAFEED_ID); + } + + @SuppressWarnings("unchecked") + private void waitForJobToBeAssigned(String jobId) throws Exception { + assertBusy(() -> { + Request getJobStats = new Request("GET", "_ml/anomaly_detectors/" + jobId + "/_stats"); + Response response = client().performRequest(getJobStats); + + Map stats = entityAsMap(response); + List> jobStats = + (List>) XContentMapValues.extractValue("jobs", stats); + + assertEquals(jobId, XContentMapValues.extractValue("job_id", jobStats.get(0))); + assertEquals("opened", XContentMapValues.extractValue("state", jobStats.get(0))); + assertThat((String) XContentMapValues.extractValue("assignment_explanation", jobStats.get(0)), isEmptyOrNullString()); + assertNotNull(XContentMapValues.extractValue("node", jobStats.get(0))); + }, 30, TimeUnit.SECONDS); + } + + @SuppressWarnings("unchecked") + private void waitForDatafeedToBeAssigned(String datafeedId) throws Exception { + assertBusy(() -> { + Request getDatafeedStats = new Request("GET", "_ml/datafeeds/" + datafeedId + "/_stats"); + Response response = client().performRequest(getDatafeedStats); + Map stats = entityAsMap(response); + List> datafeedStats = + (List>) XContentMapValues.extractValue("datafeeds", stats); + + assertEquals(datafeedId, XContentMapValues.extractValue("datafeed_id", datafeedStats.get(0))); + assertEquals("started", XContentMapValues.extractValue("state", datafeedStats.get(0))); + assertThat((String) XContentMapValues.extractValue("assignment_explanation", datafeedStats.get(0)), isEmptyOrNullString()); + assertNotNull(XContentMapValues.extractValue("node", datafeedStats.get(0))); + }, 30, TimeUnit.SECONDS); + } + + @SuppressWarnings("unchecked") + private void waitForMigration(List expectedMigratedJobs, List expectedMigratedDatafeeds, + List unMigratedJobs, List unMigratedDatafeeds) throws Exception { + assertBusy(() -> { + // wait for the eligible configs to be moved from the clusterstate + Request getClusterState = new Request("GET", "/_cluster/state/metadata"); + Response response = client().performRequest(getClusterState); + Map responseMap = entityAsMap(response); + + List> jobs = + (List>) XContentMapValues.extractValue("metadata.ml.jobs", responseMap); + assertNotNull(jobs); + + for (String jobId : expectedMigratedJobs) { + assertJob(jobId, jobs, false); + } + + for (String jobId : unMigratedJobs) { + assertJob(jobId, jobs, true); + } + + List> datafeeds = + (List>) XContentMapValues.extractValue("metadata.ml.datafeeds", responseMap); + assertNotNull(datafeeds); + + for (String datafeedId : expectedMigratedDatafeeds) { + assertDatafeed(datafeedId, datafeeds, false); + } + + for (String datafeedId : unMigratedDatafeeds) { + assertDatafeed(datafeedId, datafeeds, true); + } + + }, 30, TimeUnit.SECONDS); + } + + private void assertDatafeed(String datafeedId, List> datafeeds, boolean expectedToBePresent) { + Optional config = datafeeds.stream().map(map -> map.get("datafeed_id")) + .filter(id -> id.equals(datafeedId)).findFirst(); + if (expectedToBePresent) { + assertTrue(config.isPresent()); + } else { + assertFalse(config.isPresent()); + } + } + + private void assertJob(String jobId, List> jobs, boolean expectedToBePresent) { + Optional config = jobs.stream().map(map -> map.get("job_id")) + .filter(id -> id.equals(jobId)).findFirst(); + if (expectedToBePresent) { + assertTrue(config.isPresent()); + } else { + assertFalse(config.isPresent()); + } + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index a6e6da57b8ac6..3c7a9cee45562 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -7,7 +7,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; - import org.apache.lucene.util.TimeUnits; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -29,7 +28,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa */ @Before public void waitForTemplates() throws Exception { - XPackRestTestHelper.waitForMlTemplates(client()); + XPackRestTestHelper.waitForTemplates(client(), XPackRestTestHelper.ML_POST_V660_TEMPLATES); } @Override From 7fd250ed5315d8fcf68a1a15faba0f777e0cd199 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 17 Dec 2018 13:40:36 +0000 Subject: [PATCH 33/57] [FEATURE][ML] Add cluster setting that enables/disables config migration (#36700) This commit adds a cluster settings called `xpack.ml.enable_config_migration`. The setting is `true` by default. When set to `false`, no config migration will be attempted and non-migrated resources (e.g. jobs, datafeeds) will be able to be updated normally. Relates #32905 --- .../xpack/ml/MachineLearning.java | 7 +- .../xpack/ml/MlAssignmentNotifier.java | 5 +- .../ml/MlConfigMigrationEligibilityCheck.java | 112 ++++++ .../xpack/ml/MlConfigMigrator.java | 72 +--- .../action/TransportDeleteDatafeedAction.java | 9 +- .../ml/action/TransportDeleteJobAction.java | 9 +- .../ml/action/TransportOpenJobAction.java | 8 +- .../TransportRevertModelSnapshotAction.java | 9 +- .../action/TransportStartDatafeedAction.java | 9 +- .../action/TransportUpdateDatafeedAction.java | 9 +- .../xpack/ml/job/JobManager.java | 6 +- ...lConfigMigrationEligibilityCheckTests.java | 319 ++++++++++++++++++ .../xpack/ml/MlConfigMigratorTests.java | 122 ------- .../ml/integration/MlConfigMigratorIT.java | 73 +++- .../xpack/ml/job/JobManagerTests.java | 17 +- 15 files changed, 568 insertions(+), 218 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 8177f813eaa7e..946e654034efc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -267,7 +267,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu public static final Setting MAX_MACHINE_MEMORY_PERCENT = Setting.intSetting("xpack.ml.max_machine_memory_percent", 30, 5, 90, Property.Dynamic, Property.NodeScope); public static final Setting MAX_LAZY_ML_NODES = - Setting.intSetting("xpack.ml.max_lazy_ml_nodes", 0, 0, 3, Property.Dynamic, Property.NodeScope); + Setting.intSetting("xpack.ml.max_lazy_ml_nodes", 0, 0, 3, Property.Dynamic, Property.NodeScope); private static final Logger logger = LogManager.getLogger(XPackPlugin.class); @@ -303,7 +303,8 @@ public List> getSettings() { AutodetectBuilder.MAX_ANOMALY_RECORDS_SETTING_DYNAMIC, AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE, AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE, - AutodetectProcessManager.MIN_DISK_SPACE_OFF_HEAP)); + AutodetectProcessManager.MIN_DISK_SPACE_OFF_HEAP, + MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION)); } public Settings additionalSettings() { @@ -439,7 +440,7 @@ public Collection createComponents(Client client, ClusterService cluster jobDataCountsPersister, datafeedManager, auditor, - new MlAssignmentNotifier(auditor, threadPool, client, clusterService), + new MlAssignmentNotifier(settings, auditor, threadPool, client, clusterService), memoryTracker ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index ead55aa10cbd4..2a850ce9c14dc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -37,10 +38,10 @@ public class MlAssignmentNotifier implements ClusterStateListener, LocalNodeMast private final ThreadPool threadPool; private final AtomicBoolean enabled = new AtomicBoolean(false); - MlAssignmentNotifier(Auditor auditor, ThreadPool threadPool, Client client, ClusterService clusterService) { + MlAssignmentNotifier(Settings settings, Auditor auditor, ThreadPool threadPool, Client client, ClusterService clusterService) { this.auditor = auditor; this.clusterService = clusterService; - this.mlConfigMigrator = new MlConfigMigrator(client, clusterService); + this.mlConfigMigrator = new MlConfigMigrator(settings, client, clusterService); this.threadPool = threadPool; clusterService.addLocalNodeMasterListener(this); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java new file mode 100644 index 0000000000000..0f127919ac3d0 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.MlMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.job.config.Job; + +/** + * Checks whether migration can start and whether ML resources (e.g. jobs, datafeeds) + * are eligible to be migrated from the cluster state into the config index + */ +public class MlConfigMigrationEligibilityCheck { + + private static final Version MIN_NODE_VERSION = Version.V_6_6_0; + + public static final Setting ENABLE_CONFIG_MIGRATION = Setting.boolSetting( + "xpack.ml.enable_config_migration", true, Setting.Property.Dynamic, Setting.Property.NodeScope); + + private volatile boolean isConfigMigrationEnabled; + + public MlConfigMigrationEligibilityCheck(Settings settings, ClusterService clusterService) { + isConfigMigrationEnabled = ENABLE_CONFIG_MIGRATION.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLE_CONFIG_MIGRATION, this::setConfigMigrationEnabled); + } + + private void setConfigMigrationEnabled(boolean configMigrationEnabled) { + this.isConfigMigrationEnabled = configMigrationEnabled; + } + + /** + * Can migration start? Returns: + * False if config migration is disabled via the setting {@link #ENABLE_CONFIG_MIGRATION} + * False if the min node version of the cluster is before {@link #MIN_NODE_VERSION} + * True otherwise + * @param clusterState The cluster state + * @return A boolean that dictates if config migration can start + */ + public boolean canStartMigration(ClusterState clusterState) { + if (isConfigMigrationEnabled == false) { + return false; + } + + Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); + if (minNodeVersion.before(MIN_NODE_VERSION)) { + return false; + } + return true; + } + + /** + * Is the job a eligible for migration? Returns: + * False if {@link #canStartMigration(ClusterState)} returns {@code false} + * False if the {@link Job#isDeleting()} + * False if the job has a persistent task + * True otherwise i.e. the job is present, not deleting + * and does not have a persistent task. + * + * @param jobId The job Id + * @param clusterState The cluster state + * @return A boolean depending on the conditions listed above + */ + public boolean jobIsEligibleForMigration(String jobId, ClusterState clusterState) { + if (canStartMigration(clusterState) == false) { + return false; + } + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + Job job = mlMetadata.getJobs().get(jobId); + + if (job == null || job.isDeleting()) { + return false; + } + + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + return MlTasks.openJobIds(persistentTasks).contains(jobId) == false; + } + + /** + * Is the datafeed a eligible for migration? Returns: + * False if {@link #canStartMigration(ClusterState)} returns {@code false} + * False if the datafeed is not in the cluster state + * False if the datafeed has a persistent task + * True otherwise i.e. the datafeed is present and does not have a persistent task. + * + * @param datafeedId The datafeed Id + * @param clusterState The cluster state + * @return A boolean depending on the conditions listed above + */ + public boolean datafeedIsEligibleForMigration(String datafeedId, ClusterState clusterState) { + if (canStartMigration(clusterState) == false) { + return false; + } + + MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); + if (mlMetadata.getDatafeeds().containsKey(datafeedId) == false) { + return false; + } + + PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + return MlTasks.startedDatafeedIds(persistentTasks).contains(datafeedId) == false; + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index 2490b820440bb..e400fe0df4b93 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -80,18 +82,19 @@ public class MlConfigMigrator { private static final Logger logger = LogManager.getLogger(MlConfigMigrator.class); public static final String MIGRATED_FROM_VERSION = "migrated from version"; - public static final Version MIN_NODE_VERSION = Version.V_6_6_0; static final int MAX_BULK_WRITE_SIZE = 100; private final Client client; private final ClusterService clusterService; + private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck; private final AtomicBoolean migrationInProgress; - public MlConfigMigrator(Client client, ClusterService clusterService) { - this.client = client; - this.clusterService = clusterService; + public MlConfigMigrator(Settings settings, Client client, ClusterService clusterService) { + this.client = Objects.requireNonNull(client); + this.clusterService = Objects.requireNonNull(clusterService); + this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); this.migrationInProgress = new AtomicBoolean(false); } @@ -114,9 +117,8 @@ public MlConfigMigrator(Client client, ClusterService clusterService) { */ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener listener) { - Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); - if (minNodeVersion.before(MIN_NODE_VERSION)) { - listener.onResponse(Boolean.FALSE); + if (migrationEligibilityCheck.canStartMigration(clusterState) == false) { + listener.onResponse(false); return; } @@ -454,60 +456,4 @@ static List filterFailedDatafeedConfigWrites(Set failedDocumentI .filter(id -> failedDocumentIds.contains(DatafeedConfig.documentId(id)) == false) .collect(Collectors.toList()); } - - /** - * Is the job a eligible for migration? Returns: - * False if the min node version of the cluster is before {@link #MIN_NODE_VERSION} - * False if the job is not in the cluster state - * False if the {@link Job#isDeleting()} - * False if the job has a persistent task - * True otherwise i.e. the job is present, not deleting - * and does not have a persistent task. - * - * @param jobId The job Id - * @param clusterState clusterstate - * @return A boolean depending on the conditions listed above - */ - public static boolean jobIsEligibleForMigration(String jobId, ClusterState clusterState) { - Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); - if (minNodeVersion.before(MIN_NODE_VERSION)) { - return false; - } - - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - Job job = mlMetadata.getJobs().get(jobId); - - if (job == null || job.isDeleting()) { - return false; - } - - PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); - return MlTasks.openJobIds(persistentTasks).contains(jobId) == false; - } - - /** - * Is the datafeed a eligible for migration? Returns: - * False if the min node version of the cluster is before {@link #MIN_NODE_VERSION} - * False if the datafeed is not in the cluster state - * False if the datafeed has a persistent task - * True otherwise i.e. the datafeed is present and does not have a persistent task. - * - * @param datafeedId The datafeed Id - * @param clusterState clusterstate - * @return A boolean depending on the conditions listed above - */ - public static boolean datafeedIsEligibleForMigration(String datafeedId, ClusterState clusterState) { - Version minNodeVersion = clusterState.nodes().getMinNodeVersion(); - if (minNodeVersion.before(MIN_NODE_VERSION)) { - return false; - } - - MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState); - if (mlMetadata.getDatafeeds().containsKey(datafeedId) == false) { - return false; - } - - PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); - return MlTasks.startedDatafeedIds(persistentTasks).contains(datafeedId) == false; - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java index 42be2c7b9ddcd..97e437ec4caf2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksService; @@ -28,7 +29,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -40,9 +41,10 @@ public class TransportDeleteDatafeedAction extends TransportMasterNodeAction listener) { - if (MlConfigMigrator.datafeedIsEligibleForMigration(request.getDatafeedId(), state)) { + if (migrationEligibilityCheck.datafeedIsEligibleForMigration(request.getDatafeedId(), state)) { listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("delete datafeed", request.getDatafeedId())); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 1c4505e93afbb..481a144d7fed0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -63,7 +64,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; @@ -96,6 +97,7 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction>> listenersByJobId; @Inject - public TransportDeleteJobAction(TransportService transportService, ClusterService clusterService, + public TransportDeleteJobAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, PersistentTasksService persistentTasksService, Client client, Auditor auditor, JobResultsProvider jobResultsProvider, @@ -121,6 +123,7 @@ public TransportDeleteJobAction(TransportService transportService, ClusterServic this.jobConfigProvider = jobConfigProvider; this.datafeedConfigProvider = datafeedConfigProvider; this.memoryTracker = memoryTracker; + this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); this.listenersByJobId = new HashMap<>(); } @@ -148,7 +151,7 @@ protected void masterOperation(DeleteJobAction.Request request, ClusterState sta protected void masterOperation(Task task, DeleteJobAction.Request request, ClusterState state, ActionListener listener) { - if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobId(), state)) { + if (migrationEligibilityCheck.jobIsEligibleForMigration(request.getJobId(), state)) { listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("delete job", request.getJobId())); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 433d1b420303f..557ebd8f87b04 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -67,7 +67,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -107,9 +107,10 @@ public class TransportOpenJobAction extends TransportMasterNodeAction listener) { - if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobParams().getJobId(), state)) { + if (migrationEligibilityCheck.jobIsEligibleForMigration(request.getJobParams().getJobId(), state)) { listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("open job", request.getJobParams().getJobId())); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index cf15f6ab41a66..a940d6666c9fd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -25,7 +26,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; @@ -41,9 +42,10 @@ public class TransportRevertModelSnapshotAction extends TransportMasterNodeActio private final JobManager jobManager; private final JobResultsProvider jobResultsProvider; private final JobDataCountsPersister jobDataCountsPersister; + private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck; @Inject - public TransportRevertModelSnapshotAction(ThreadPool threadPool, TransportService transportService, + public TransportRevertModelSnapshotAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, JobResultsProvider jobResultsProvider, ClusterService clusterService, Client client, JobDataCountsPersister jobDataCountsPersister) { @@ -53,6 +55,7 @@ public TransportRevertModelSnapshotAction(ThreadPool threadPool, TransportServic this.jobManager = jobManager; this.jobResultsProvider = jobResultsProvider; this.jobDataCountsPersister = jobDataCountsPersister; + this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); } @Override @@ -68,7 +71,7 @@ protected RevertModelSnapshotAction.Response newResponse() { @Override protected void masterOperation(RevertModelSnapshotAction.Request request, ClusterState state, ActionListener listener) { - if (MlConfigMigrator.jobIsEligibleForMigration(request.getJobId(), state)) { + if (migrationEligibilityCheck.jobIsEligibleForMigration(request.getJobId(), state)) { listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("revert model snapshot", request.getJobId())); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index 9fa0f5a5acb87..5867948bbad63 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.RemoteClusterLicenseChecker; @@ -44,7 +45,7 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.DatafeedNodeSelector; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; @@ -76,9 +77,10 @@ public class TransportStartDatafeedAction extends TransportMasterNodeAction listener) throws Exception { - if (MlConfigMigrator.datafeedIsEligibleForMigration(request.getUpdate().getId(), state)) { + if (migrationEligibilityCheck.datafeedIsEligibleForMigration(request.getUpdate().getId(), state)) { listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("update datafeed", request.getUpdate().getId())); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 291347028453b..33047c1fca39a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -48,7 +48,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.MlConfigMigrator; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; @@ -92,6 +92,7 @@ public class JobManager { private final ThreadPool threadPool; private final UpdateJobProcessNotifier updateJobProcessNotifier; private final JobConfigProvider jobConfigProvider; + private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck; private volatile ByteSizeValue maxModelMemoryLimit; @@ -109,6 +110,7 @@ public JobManager(Environment environment, Settings settings, JobResultsProvider this.threadPool = Objects.requireNonNull(threadPool); this.updateJobProcessNotifier = updateJobProcessNotifier; this.jobConfigProvider = new JobConfigProvider(client); + this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); maxModelMemoryLimit = MachineLearningField.MAX_MODEL_MEMORY_LIMIT.get(settings); clusterService.getClusterSettings() @@ -338,7 +340,7 @@ public void updateJob(UpdateJobAction.Request request, ActionListener(Collections.singletonList( + MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION))); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + } + + private static Settings newSettings(boolean migrationEnabled) { + return Settings.builder() + .put(MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION.getKey(), migrationEnabled) + .build(); + } + + private DatafeedConfig createCompatibleDatafeed(String jobId) { + // create a datafeed without aggregations or anything + // else that may cause validation errors + DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("df-" + jobId, jobId); + datafeedBuilder.setIndices(Collections.singletonList("my_index")); + return datafeedBuilder.build(); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java index 92d06724a0488..d9ea035e58234 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java @@ -11,9 +11,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.MlMetadata; @@ -24,7 +21,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobTests; -import java.net.InetAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -218,124 +214,6 @@ public void testRemoveJobsAndDatafeeds_removeSome() { assertThat(removalResult.removedDatafeedIds, empty()); } - public void testJobIsEligibleForMigration_givenNodesNotUpToVersion() { - // mixed 6.5 and 6.6 nodes - ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0)) - .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0))) - .build(); - - assertFalse(MlConfigMigrator.jobIsEligibleForMigration("pre-min-version", clusterState)); - } - - public void testJobIsEligibleForMigration_givenJobNotInClusterState() { - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")).build(); - assertFalse(MlConfigMigrator.jobIsEligibleForMigration("not-in-state", clusterState)); - } - - public void testJobIsEligibleForMigration_givenDeletingJob() { - Job deletingJob = JobTests.buildJobBuilder("deleting-job").setDeleting(true).build(); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder().putJob(deletingJob, false); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.jobTaskId(deletingJob.getId()), - MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(deletingJob.getId()), - new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); - - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - ) - .build(); - - assertFalse(MlConfigMigrator.jobIsEligibleForMigration(deletingJob.getId(), clusterState)); - } - - public void testJobIsEligibleForMigration_givenOpenJob() { - Job openJob = JobTests.buildJobBuilder("open-job").build(); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder().putJob(openJob, false); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.jobTaskId(openJob.getId()), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(openJob.getId()), - new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); - - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - ) - .build(); - - assertFalse(MlConfigMigrator.jobIsEligibleForMigration(openJob.getId(), clusterState)); - } - - public void testJobIsEligibleForMigration_givenClosedJob() { - Job closedJob = JobTests.buildJobBuilder("closed-job").build(); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder().putJob(closedJob, false); - - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build()) - ) - .build(); - - assertTrue(MlConfigMigrator.jobIsEligibleForMigration(closedJob.getId(), clusterState)); - } - - public void testDatafeedIsEligibleForMigration_givenNodesNotUpToVersion() { - // mixed 6.5 and 6.6 nodes - ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .nodes(DiscoveryNodes.builder() - .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0)) - .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0))) - .build(); - - assertFalse(MlConfigMigrator.datafeedIsEligibleForMigration("pre-min-version", clusterState)); - } - - public void testDatafeedIsEligibleForMigration_givenDatafeedNotInClusterState() { - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")).build(); - assertFalse(MlConfigMigrator.datafeedIsEligibleForMigration("not-in-state", clusterState)); - } - - public void testDatafeedIsEligibleForMigration_givenStartedDatafeed() { - Job openJob = JobTests.buildJobBuilder("open-job").build(); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder().putJob(openJob, false); - mlMetadata.putDatafeed(createCompatibleDatafeed(openJob.getId()), Collections.emptyMap()); - String datafeedId = "df-" + openJob.getId(); - - PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - tasksBuilder.addTask(MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, - new StartDatafeedAction.DatafeedParams(datafeedId, 0L), - new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); - - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build()) - .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) - ) - .build(); - - assertFalse(MlConfigMigrator.datafeedIsEligibleForMigration(datafeedId, clusterState)); - } - - public void testDatafeedIsEligibleForMigration_givenStoppedDatafeed() { - Job job = JobTests.buildJobBuilder("closed-job").build(); - MlMetadata.Builder mlMetadata = new MlMetadata.Builder().putJob(job, false); - mlMetadata.putDatafeed(createCompatibleDatafeed(job.getId()), Collections.emptyMap()); - String datafeedId = "df-" + job.getId(); - - ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build()) - ) - .build(); - - assertTrue(MlConfigMigrator.datafeedIsEligibleForMigration(datafeedId, clusterState)); - } - public void testLimitWrites_GivenBelowLimit() { MlConfigMigrator.JobsAndDatafeeds jobsAndDatafeeds = MlConfigMigrator.limitWrites(Collections.emptyList(), Collections.emptyMap()); assertThat(jobsAndDatafeeds.datafeedConfigs, empty()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java index 51a3b5d2366b0..b81805fb3fbdf 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java @@ -11,16 +11,21 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; +import org.junit.Before; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -28,13 +33,25 @@ import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MlConfigMigratorIT extends MlSingleNodeTestCase { + private ClusterService clusterService; + + @Before + public void setUpTests() { + clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings(nodeSettings(), new HashSet<>(Collections.singletonList( + MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION))); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + } + public void testWriteConfigToIndex() throws InterruptedException { final String indexJobId = "job-already-migrated"; @@ -50,8 +67,7 @@ public void testWriteConfigToIndex() throws InterruptedException { // put a job representing a previously migrated job blockingCall(actionListener -> jobConfigProvider.putJob(migratedJob, actionListener), indexResponseHolder, exceptionHolder); - ClusterService clusterService = mock(ClusterService.class); - MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(client(), clusterService); + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(nodeSettings(), client(), clusterService); AtomicReference> failedIdsHolder = new AtomicReference<>(); Job foo = buildJobBuilder("foo").build(); @@ -97,7 +113,6 @@ public void testMigrateConfigs() throws InterruptedException { .putCustom(MlMetadata.TYPE, mlMetadata.build())) .build(); - ClusterService clusterService = mock(ClusterService.class); doAnswer(invocation -> { ClusterStateUpdateTask listener = (ClusterStateUpdateTask) invocation.getArguments()[1]; listener.clusterStateProcessed("source", mock(ClusterState.class), mock(ClusterState.class)); @@ -108,7 +123,7 @@ public void testMigrateConfigs() throws InterruptedException { AtomicReference responseHolder = new AtomicReference<>(); // do the migration - MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(client(), clusterService); + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(nodeSettings(), client(), clusterService); blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), responseHolder, exceptionHolder); @@ -138,6 +153,56 @@ public void testMigrateConfigs() throws InterruptedException { assertThat(datafeedsHolder.get(), hasSize(1)); assertEquals("df-1", datafeedsHolder.get().get(0).getId()); } + + public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws InterruptedException { + Settings settings = Settings.builder().put(nodeSettings()) + .put(MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION.getKey(), false) + .build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(Collections.singletonList( + MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION))); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + + // and jobs and datafeeds clusterstate + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + mlMetadata.putJob(buildJobBuilder("job-foo").build(), false); + mlMetadata.putJob(buildJobBuilder("job-bar").build(), false); + DatafeedConfig.Builder builder = new DatafeedConfig.Builder("df-1", "job-foo"); + builder.setIndices(Collections.singletonList("beats*")); + mlMetadata.putDatafeed(builder.build(), Collections.emptyMap()); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + + // do the migration + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(settings, client(), clusterService); + blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), + responseHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertFalse(responseHolder.get()); + + // check the jobs have not been migrated + AtomicReference> jobsHolder = new AtomicReference<>(); + JobConfigProvider jobConfigProvider = new JobConfigProvider(client()); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, true, actionListener), + jobsHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(jobsHolder.get().isEmpty(), is(true)); + + // check datafeeds have not been migrated + DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client(), xContentRegistry()); + AtomicReference> datafeedsHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", true, actionListener), + datafeedsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(datafeedsHolder.get().isEmpty(), is(true)); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index 5341d0466df63..a24950ed0918e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; @@ -59,6 +60,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.TreeSet; @@ -94,10 +96,13 @@ public class JobManagerTests extends ESTestCase { @Before public void setup() throws Exception { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .build(); environment = TestEnvironment.newEnvironment(settings); analysisRegistry = CategorizationAnalyzerTests.buildTestAnalysisRegistry(environment); clusterService = mock(ClusterService.class); + givenClusterSettings(settings); jobResultsProvider = mock(JobResultsProvider.class); auditor = mock(Auditor.class); @@ -550,9 +555,6 @@ private Job.Builder createJob() { } private JobManager createJobManager(Client client) { - ClusterSettings clusterSettings = new ClusterSettings(environment.settings(), - Collections.singleton(MachineLearningField.MAX_MODEL_MEMORY_LIMIT)); - when(clusterService.getClusterSettings()).thenReturn(clusterSettings); return new JobManager(environment, environment.settings(), jobResultsProvider, clusterService, auditor, threadPool, client, updateJobProcessNotifier); } @@ -569,4 +571,11 @@ private BytesReference toBytesReference(ToXContent content) throws IOException { return BytesReference.bytes(xContentBuilder); } } + + private void givenClusterSettings(Settings settings) { + ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(Arrays.asList( + MachineLearningField.MAX_MODEL_MEMORY_LIMIT, + MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION))); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + } } From 4808c651656e4772a8c24e4f493accbd01877600 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 17 Dec 2018 16:38:24 +0000 Subject: [PATCH 34/57] [ML] Snapshot ml configs before migrating (#36645) --- .../xpack/ml/MlAssignmentNotifier.java | 56 +++++---------- .../xpack/ml/MlConfigMigrator.java | 68 +++++++++++++++++- .../xpack/ml/MlAssignmentNotifierTests.java | 72 +++++++++++++------ .../ml/integration/MlConfigMigratorIT.java | 40 +++++++++-- 4 files changed, 165 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index 2a850ce9c14dc..4dc3873c5859d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -10,9 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -26,75 +24,57 @@ import org.elasticsearch.xpack.ml.notifications.Auditor; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; -public class MlAssignmentNotifier implements ClusterStateListener, LocalNodeMasterListener { +public class MlAssignmentNotifier implements ClusterStateListener { private static final Logger logger = LogManager.getLogger(MlAssignmentNotifier.class); private final Auditor auditor; - private final ClusterService clusterService; private final MlConfigMigrator mlConfigMigrator; private final ThreadPool threadPool; - private final AtomicBoolean enabled = new AtomicBoolean(false); MlAssignmentNotifier(Settings settings, Auditor auditor, ThreadPool threadPool, Client client, ClusterService clusterService) { this.auditor = auditor; - this.clusterService = clusterService; this.mlConfigMigrator = new MlConfigMigrator(settings, client, clusterService); this.threadPool = threadPool; - clusterService.addLocalNodeMasterListener(this); + clusterService.addListener(this); } MlAssignmentNotifier(Auditor auditor, ThreadPool threadPool, MlConfigMigrator mlConfigMigrator, ClusterService clusterService) { this.auditor = auditor; - this.clusterService = clusterService; this.mlConfigMigrator = mlConfigMigrator; this.threadPool = threadPool; - clusterService.addLocalNodeMasterListener(this); + clusterService.addListener(this); } - @Override - public void onMaster() { - if (enabled.compareAndSet(false, true)) { - clusterService.addListener(this); - } - } - - @Override - public void offMaster() { - if (enabled.compareAndSet(true, false)) { - clusterService.removeListener(this); - } - } - - @Override - public String executorName() { + private String executorName() { return ThreadPool.Names.GENERIC; } @Override public void clusterChanged(ClusterChangedEvent event) { - if (enabled.get() == false) { - return; - } - if (event.metaDataChanged() == false) { + + if (event.localNodeMaster() == false) { return; } - PersistentTasksCustomMetaData previous = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - PersistentTasksCustomMetaData current = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); mlConfigMigrator.migrateConfigsWithoutTasks(event.state(), ActionListener.wrap( - response -> threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())), + response -> threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(event)), e -> { logger.error("error migrating ml configurations", e); - threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(current, previous, event.state())); + threadPool.executor(executorName()).execute(() -> auditChangesToMlTasks(event)); } )); } - private void auditChangesToMlTasks(PersistentTasksCustomMetaData current, PersistentTasksCustomMetaData previous, - ClusterState state) { + private void auditChangesToMlTasks(ClusterChangedEvent event) { + + if (event.metaDataChanged() == false) { + return; + } + + PersistentTasksCustomMetaData previous = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData current = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (Objects.equals(previous, current)) { return; @@ -112,7 +92,7 @@ private void auditChangesToMlTasks(PersistentTasksCustomMetaData current, Persis if (currentAssignment.getExecutorNode() == null) { auditor.warning(jobId, "No node found to open job. Reasons [" + currentAssignment.getExplanation() + "]"); } else { - DiscoveryNode node = state.nodes().get(currentAssignment.getExecutorNode()); + DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); auditor.info(jobId, "Opening job on node [" + node.toString() + "]"); } } else if (MlTasks.DATAFEED_TASK_NAME.equals(currentTask.getTaskName())) { @@ -126,7 +106,7 @@ private void auditChangesToMlTasks(PersistentTasksCustomMetaData current, Persis auditor.warning(jobId, msg); } } else { - DiscoveryNode node = state.nodes().get(currentAssignment.getExecutorNode()); + DiscoveryNode node = event.state().nodes().get(currentAssignment.getExecutorNode()); if (jobId != null) { auditor.info(jobId, "Starting datafeed [" + datafeedParams.getDatafeedId() + "] on node [" + node + "]"); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index e400fe0df4b93..e17c23da0686e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -9,10 +9,14 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; @@ -31,12 +35,14 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -90,12 +96,14 @@ public class MlConfigMigrator { private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck; private final AtomicBoolean migrationInProgress; + private final AtomicBoolean firstTime; public MlConfigMigrator(Settings settings, Client client, ClusterService clusterService) { this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); this.migrationInProgress = new AtomicBoolean(false); + this.firstTime = new AtomicBoolean(true); } /** @@ -145,8 +153,23 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener } ); + if (firstTime.get()) { + snapshotMlMeta(MlMetadata.getMlMetadata(clusterState), ActionListener.wrap( + response -> { + firstTime.set(false); + migrate(jobsAndDatafeedsToMigrate, unMarkMigrationInProgress); + }, + unMarkMigrationInProgress::onFailure + )); + return; + } + + migrate(jobsAndDatafeedsToMigrate, unMarkMigrationInProgress); + } + + private void migrate(JobsAndDatafeeds jobsAndDatafeedsToMigrate, ActionListener listener) { if (jobsAndDatafeedsToMigrate.totalCount() == 0) { - unMarkMigrationInProgress.onResponse(Boolean.FALSE); + listener.onResponse(Boolean.FALSE); return; } @@ -157,9 +180,9 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.jobs); List successfulDatafeedWrites = filterFailedDatafeedConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.datafeedConfigs); - removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, unMarkMigrationInProgress); + removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, listener); }, - unMarkMigrationInProgress::onFailure + listener::onFailure )); } @@ -299,6 +322,45 @@ private IndexRequest indexRequest(ToXContentObject source, String documentId, To return indexRequest; } + + // public for testing + public void snapshotMlMeta(MlMetadata mlMetadata, ActionListener listener) { + + if (mlMetadata.getJobs().isEmpty() && mlMetadata.getDatafeeds().isEmpty()) { + listener.onResponse(Boolean.TRUE); + return; + } + + logger.debug("taking a snapshot of mlmetadata"); + String documentId = "ml-config"; + IndexRequestBuilder indexRequest = client.prepareIndex(AnomalyDetectorsIndex.jobStateIndexName(), + ElasticsearchMappings.DOC_TYPE, documentId) + .setOpType(DocWriteRequest.OpType.CREATE); + + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + mlMetadata.toXContent(builder, params); + builder.endObject(); + + indexRequest.setSource(builder); + } catch (IOException e) { + logger.error("failed to serialise mlmetadata", e); + listener.onFailure(e); + return; + } + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest.request(), + ActionListener.wrap( + indexResponse -> { + listener.onResponse(indexResponse.getResult() == DocWriteResponse.Result.CREATED); + }, + listener::onFailure), + client::index + ); + } + + public static Job updateJobForMigration(Job job) { Job.Builder builder = new Job.Builder(job); Map custom = job.getCustomSettings() == null ? new HashMap<>() : new HashMap<>(job.getCustomSettings()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java index b77ed582709ca..5c8c253794794 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java @@ -31,7 +31,6 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -69,34 +68,39 @@ private void setupMocks() { public void testClusterChanged_info() { MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); - notifier.onMaster(); - DiscoveryNode node = - new DiscoveryNode("node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.CURRENT); ClusterState previous = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, new PersistentTasksCustomMetaData(0L, Collections.emptyMap()))) .build(); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); - addJobTask("job_id", "node_id", null, tasksBuilder); + addJobTask("job_id", "_node_id", null, tasksBuilder); MetaData metaData = MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()).build(); - ClusterState state = ClusterState.builder(new ClusterName("_name")) + ClusterState newState = ClusterState.builder(new ClusterName("_name")) .metaData(metaData) - .nodes(DiscoveryNodes.builder().add(node)) + // set local node master + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) .build(); - notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); verify(auditor, times(1)).info(eq("job_id"), any()); - verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(state), any()); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(newState), any()); - notifier.offMaster(); - notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); + // no longer master + newState = ClusterState.builder(new ClusterName("_name")) + .metaData(metaData) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.CURRENT))) + .build(); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); verifyNoMoreInteractions(auditor); } public void testClusterChanged_warning() { MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); - notifier.onMaster(); ClusterState previous = ClusterState.builder(new ClusterName("_name")) .metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, @@ -106,21 +110,31 @@ public void testClusterChanged_warning() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", null, null, tasksBuilder); MetaData metaData = MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()).build(); - ClusterState state = ClusterState.builder(new ClusterName("_name")) + ClusterState newState = ClusterState.builder(new ClusterName("_name")) .metaData(metaData) + // set local node master + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) .build(); - notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); verify(auditor, times(1)).warning(eq("job_id"), any()); - verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(state), any()); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(eq(newState), any()); + + // no longer master + newState = ClusterState.builder(new ClusterName("_name")) + .metaData(metaData) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT))) + .build(); - notifier.offMaster(); - notifier.clusterChanged(new ClusterChangedEvent("_test", state, previous)); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); verifyNoMoreInteractions(auditor); } public void testClusterChanged_noPersistentTaskChanges() { MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, threadPool, configMigrator, clusterService); - notifier.onMaster(); PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask("job_id", null, null, tasksBuilder); @@ -129,14 +143,26 @@ public void testClusterChanged_noPersistentTaskChanges() { .metaData(metaData) .build(); - ClusterState current = ClusterState.builder(new ClusterName("_name")) + ClusterState newState = ClusterState.builder(new ClusterName("_name")) .metaData(metaData) + // set local node master + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) .build(); - notifier.clusterChanged(new ClusterChangedEvent("_test", current, previous)); - verify(configMigrator, never()).migrateConfigsWithoutTasks(any(), any()); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); + verify(configMigrator, times(1)).migrateConfigsWithoutTasks(any(), any()); + verifyNoMoreInteractions(auditor); - notifier.offMaster(); - verify(configMigrator, never()).migrateConfigsWithoutTasks(any(), any()); + // no longer master + newState = ClusterState.builder(new ClusterName("_name")) + .metaData(metaData) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT))) + .build(); + notifier.clusterChanged(new ClusterChangedEvent("_test", newState, previous)); + verifyNoMoreInteractions(configMigrator); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java index b81805fb3fbdf..1dc06e0e2aef6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -13,9 +14,16 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck; import org.elasticsearch.xpack.ml.MlConfigMigrator; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; @@ -23,6 +31,8 @@ import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; import org.junit.Before; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -98,7 +108,7 @@ public void testWriteConfigToIndex() throws InterruptedException { assertNull(alreadyMigratedJob.getCustomSettings()); } - public void testMigrateConfigs() throws InterruptedException { + public void testMigrateConfigs() throws InterruptedException, IOException { // and jobs and datafeeds clusterstate MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); @@ -124,11 +134,13 @@ public void testMigrateConfigs() throws InterruptedException { // do the migration MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(nodeSettings(), client(), clusterService); + // the first time this is called mlmetadata will be snap-shotted blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), responseHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertTrue(responseHolder.get()); + assertSnapshot(mlMetadata.build()); // check the jobs have been migrated AtomicReference> jobsHolder = new AtomicReference<>(); @@ -171,9 +183,9 @@ public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws Int mlMetadata.putDatafeed(builder.build(), Collections.emptyMap()); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder() - .putCustom(MlMetadata.TYPE, mlMetadata.build())) - .build(); + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); AtomicReference exceptionHolder = new AtomicReference<>(); AtomicReference responseHolder = new AtomicReference<>(); @@ -181,7 +193,7 @@ public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws Int // do the migration MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(settings, client(), clusterService); blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), - responseHolder, exceptionHolder); + responseHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertFalse(responseHolder.get()); @@ -190,7 +202,7 @@ public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws Int AtomicReference> jobsHolder = new AtomicReference<>(); JobConfigProvider jobConfigProvider = new JobConfigProvider(client()); blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, true, actionListener), - jobsHolder, exceptionHolder); + jobsHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertThat(jobsHolder.get().isEmpty(), is(true)); @@ -198,11 +210,25 @@ public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws Int DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client(), xContentRegistry()); AtomicReference> datafeedsHolder = new AtomicReference<>(); blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", true, actionListener), - datafeedsHolder, exceptionHolder); + datafeedsHolder, exceptionHolder); assertNull(exceptionHolder.get()); assertThat(datafeedsHolder.get().isEmpty(), is(true)); } + + public void assertSnapshot(MlMetadata expectedMlMetadata) throws IOException { + GetResponse getResponse = client() + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ElasticsearchMappings.DOC_TYPE, "ml-config").get(); + + assertTrue(getResponse.isExists()); + + try (InputStream stream = getResponse.getSourceAsBytesRef().streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + MlMetadata recoveredMeta = MlMetadata.LENIENT_PARSER.apply(parser, null).build(); + assertEquals(expectedMlMetadata, recoveredMeta); + } + } } From b1127431aefc8013bc2d7e32635b84894871ea47 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 17 Dec 2018 22:36:36 +0000 Subject: [PATCH 35/57] [FEATURE][ML] Split in batches and migrate all jobs and datafeeds (#36716) Relates #32905 --- .../xpack/ml/MlConfigMigrator.java | 95 +++++++++++-------- .../ml/integration/MlConfigMigratorIT.java | 77 ++++++++++++++- 2 files changed, 132 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index e17c23da0686e..c3b9626ffd042 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -38,6 +39,7 @@ import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; +import org.elasticsearch.xpack.ml.utils.ChainTaskExecutor; import java.io.IOException; import java.util.ArrayList; @@ -96,14 +98,14 @@ public class MlConfigMigrator { private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck; private final AtomicBoolean migrationInProgress; - private final AtomicBoolean firstTime; + private final AtomicBoolean tookConfigSnapshot; public MlConfigMigrator(Settings settings, Client client, ClusterService clusterService) { this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService); this.migrationInProgress = new AtomicBoolean(false); - this.firstTime = new AtomicBoolean(true); + this.tookConfigSnapshot = new AtomicBoolean(false); } /** @@ -135,12 +137,7 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener return; } - Collection stoppedDatafeeds = stoppedDatafeedConfigs(clusterState); - Map eligibleJobs = nonDeletingJobs(closedJobConfigs(clusterState)).stream() - .map(MlConfigMigrator::updateJobForMigration) - .collect(Collectors.toMap(Job::getId, Function.identity(), (a, b) -> a)); - - JobsAndDatafeeds jobsAndDatafeedsToMigrate = limitWrites(stoppedDatafeeds, eligibleJobs); + logger.debug("migrating ml configurations"); ActionListener unMarkMigrationInProgress = ActionListener.wrap( response -> { @@ -153,37 +150,36 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener } ); - if (firstTime.get()) { - snapshotMlMeta(MlMetadata.getMlMetadata(clusterState), ActionListener.wrap( - response -> { - firstTime.set(false); - migrate(jobsAndDatafeedsToMigrate, unMarkMigrationInProgress); - }, - unMarkMigrationInProgress::onFailure - )); - return; - } + snapshotMlMeta(MlMetadata.getMlMetadata(clusterState), ActionListener.wrap( + response -> { + // We have successfully snapshotted the ML configs so we don't need to try again + tookConfigSnapshot.set(true); - migrate(jobsAndDatafeedsToMigrate, unMarkMigrationInProgress); + List batches = splitInBatches(clusterState); + if (batches.isEmpty()) { + unMarkMigrationInProgress.onResponse(Boolean.FALSE); + return; + } + migrateBatches(batches, unMarkMigrationInProgress); + }, + unMarkMigrationInProgress::onFailure + )); } - private void migrate(JobsAndDatafeeds jobsAndDatafeedsToMigrate, ActionListener listener) { - if (jobsAndDatafeedsToMigrate.totalCount() == 0) { - listener.onResponse(Boolean.FALSE); - return; - } - - logger.debug("migrating ml configurations"); - - writeConfigToIndex(jobsAndDatafeedsToMigrate.datafeedConfigs, jobsAndDatafeedsToMigrate.jobs, ActionListener.wrap( + private void migrateBatches(List batches, ActionListener listener) { + ChainTaskExecutor chainTaskExecutor = new ChainTaskExecutor(EsExecutors.newDirectExecutorService(), true); + for (JobsAndDatafeeds batch : batches) { + chainTaskExecutor.add(chainedListener -> writeConfigToIndex(batch.datafeedConfigs, batch.jobs, ActionListener.wrap( failedDocumentIds -> { - List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.jobs); + List successfulJobWrites = filterFailedJobConfigWrites(failedDocumentIds, batch.jobs); List successfulDatafeedWrites = - filterFailedDatafeedConfigWrites(failedDocumentIds, jobsAndDatafeedsToMigrate.datafeedConfigs); - removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, listener); + filterFailedDatafeedConfigWrites(failedDocumentIds, batch.datafeedConfigs); + removeFromClusterState(successfulJobWrites, successfulDatafeedWrites, chainedListener); }, - listener::onFailure - )); + chainedListener::onFailure + ))); + } + chainTaskExecutor.execute(ActionListener.wrap(aVoid -> listener.onResponse(true), listener::onFailure)); } // Exposed for testing @@ -208,9 +204,9 @@ public void writeConfigToIndex(Collection datafeedsToMigrate, } private void removeFromClusterState(List jobsToRemoveIds, List datafeedsToRemoveIds, - ActionListener listener) { + ActionListener listener) { if (jobsToRemoveIds.isEmpty() && datafeedsToRemoveIds.isEmpty()) { - listener.onResponse(Boolean.FALSE); + listener.onResponse(null); return; } @@ -244,7 +240,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS logger.info("ml datafeed configurations migrated: {}", removedConfigs.get().removedDatafeedIds); } } - listener.onResponse(Boolean.TRUE); + listener.onResponse(null); } }); } @@ -326,12 +322,17 @@ private IndexRequest indexRequest(ToXContentObject source, String documentId, To // public for testing public void snapshotMlMeta(MlMetadata mlMetadata, ActionListener listener) { + if (tookConfigSnapshot.get()) { + listener.onResponse(true); + return; + } + if (mlMetadata.getJobs().isEmpty() && mlMetadata.getDatafeeds().isEmpty()) { - listener.onResponse(Boolean.TRUE); + listener.onResponse(true); return; } - logger.debug("taking a snapshot of mlmetadata"); + logger.debug("taking a snapshot of ml_metadata"); String documentId = "ml-config"; IndexRequestBuilder indexRequest = client.prepareIndex(AnomalyDetectorsIndex.jobStateIndexName(), ElasticsearchMappings.DOC_TYPE, documentId) @@ -345,7 +346,7 @@ public void snapshotMlMeta(MlMetadata mlMetadata, ActionListener listen indexRequest.setSource(builder); } catch (IOException e) { - logger.error("failed to serialise mlmetadata", e); + logger.error("failed to serialise ml_metadata", e); listener.onFailure(e); return; } @@ -437,6 +438,22 @@ public int totalCount() { } } + public static List splitInBatches(ClusterState clusterState) { + Collection stoppedDatafeeds = stoppedDatafeedConfigs(clusterState); + Map eligibleJobs = nonDeletingJobs(closedJobConfigs(clusterState)).stream() + .map(MlConfigMigrator::updateJobForMigration) + .collect(Collectors.toMap(Job::getId, Function.identity(), (a, b) -> a)); + + List batches = new ArrayList<>(); + while (stoppedDatafeeds.isEmpty() == false || eligibleJobs.isEmpty() == false) { + JobsAndDatafeeds batch = limitWrites(stoppedDatafeeds, eligibleJobs); + batches.add(batch); + stoppedDatafeeds.removeAll(batch.datafeedConfigs); + batch.jobs.forEach(job -> eligibleJobs.remove(job.getId())); + } + return batches; + } + /** * Return at most {@link #MAX_BULK_WRITE_SIZE} configs favouring * datafeed and job pairs so if a datafeed is chosen so is its job. diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java index 1dc06e0e2aef6..d98abea55535c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java @@ -109,7 +109,6 @@ public void testWriteConfigToIndex() throws InterruptedException { } public void testMigrateConfigs() throws InterruptedException, IOException { - // and jobs and datafeeds clusterstate MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); mlMetadata.putJob(buildJobBuilder("job-foo").build(), false); @@ -166,6 +165,82 @@ public void testMigrateConfigs() throws InterruptedException, IOException { assertEquals("df-1", datafeedsHolder.get().get(0).getId()); } + public void testMigrateConfigs_GivenLargeNumberOfJobsAndDatafeeds() throws InterruptedException { + int jobCount = randomIntBetween(150, 201); + int datafeedCount = randomIntBetween(150, jobCount); + + // and jobs and datafeeds clusterstate + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + for (int i = 0; i < jobCount; i++) { + mlMetadata.putJob(buildJobBuilder("job-" + i).build(), false); + } + for (int i = 0; i < datafeedCount; i++) { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder("df-" + i, "job-" + i); + builder.setIndices(Collections.singletonList("beats*")); + mlMetadata.putDatafeed(builder.build(), Collections.emptyMap()); + } + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + + doAnswer(invocation -> { + ClusterStateUpdateTask listener = (ClusterStateUpdateTask) invocation.getArguments()[1]; + listener.clusterStateProcessed("source", mock(ClusterState.class), mock(ClusterState.class)); + return null; + }).when(clusterService).submitStateUpdateTask(eq("remove-migrated-ml-configs"), any()); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + + // do the migration + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(nodeSettings(), client(), clusterService); + blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), + responseHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertTrue(responseHolder.get()); + + // check the jobs have been migrated + AtomicReference> jobsHolder = new AtomicReference<>(); + JobConfigProvider jobConfigProvider = new JobConfigProvider(client()); + blockingCall(actionListener -> jobConfigProvider.expandJobs("*", true, true, actionListener), + jobsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(jobsHolder.get(), hasSize(jobCount)); + + // check datafeeds are migrated + DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client(), xContentRegistry()); + AtomicReference> datafeedsHolder = new AtomicReference<>(); + blockingCall(actionListener -> datafeedConfigProvider.expandDatafeedConfigs("*", true, actionListener), + datafeedsHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertThat(datafeedsHolder.get(), hasSize(datafeedCount)); + } + + public void testMigrateConfigs_GivenNoJobsOrDatafeeds() throws InterruptedException { + // Add empty ML metadata + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(MlMetadata.TYPE, mlMetadata.build())) + .build(); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference responseHolder = new AtomicReference<>(); + + // do the migration + MlConfigMigrator mlConfigMigrator = new MlConfigMigrator(nodeSettings(), client(), clusterService); + blockingCall(actionListener -> mlConfigMigrator.migrateConfigsWithoutTasks(clusterState, actionListener), + responseHolder, exceptionHolder); + + assertNull(exceptionHolder.get()); + assertFalse(responseHolder.get()); + } + public void testMigrateConfigsWithoutTasks_GivenMigrationIsDisabled() throws InterruptedException { Settings settings = Settings.builder().put(nodeSettings()) .put(MlConfigMigrationEligibilityCheck.ENABLE_CONFIG_MIGRATION.getKey(), false) From 2049651b0c2b07f02cd1e88edac4731f3431345d Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Mon, 17 Dec 2018 18:58:06 +0200 Subject: [PATCH 36/57] SQL: Fix translation of LIKE/RLIKE keywords (#36672) * SQL: Fix translation of LIKE/RLIKE keywords Refactor Like/RLike functions to simplify internals and improve query translation when chained or within a script context. Fix #36039 Fix #36584 --- .../sql/qa/src/main/resources/agg.sql-spec | 2 + .../qa/src/main/resources/datetime.sql-spec | 5 +- .../sql/qa/src/main/resources/filter.sql-spec | 2 + .../whitelist/InternalSqlScriptUtils.java | 1 + .../sql/expression/predicate/regex/Like.java | 20 ++--- .../predicate/regex/LikePattern.java | 24 +----- .../sql/expression/predicate/regex/RLike.java | 21 ++--- .../predicate/regex/RegexMatch.java | 29 +++++-- .../expression/predicate/regex/RegexPipe.java | 34 -------- .../predicate/regex/RegexProcessor.java | 68 +++++++-------- .../xpack/sql/parser/ExpressionBuilder.java | 4 +- .../xpack/sql/planner/QueryTranslator.java | 83 +++++++++++++++---- .../xpack/sql/optimizer/OptimizerTests.java | 6 +- .../xpack/sql/tree/NodeSubclassTests.java | 13 ++- 14 files changed, 158 insertions(+), 154 deletions(-) delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec index 9adbe79edc685..149e23f771349 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec @@ -280,6 +280,8 @@ aggMaxWithAlias SELECT gender g, MAX(emp_no) m FROM "test_emp" GROUP BY g ORDER BY gender; aggMaxOnDate SELECT gender, MAX(birth_date) m FROM "test_emp" GROUP BY gender ORDER BY gender; +aggAvgAndMaxWithLikeFilter +SELECT CAST(AVG(salary) AS LONG) AS avg, CAST(SUM(salary) AS LONG) AS s FROM "test_emp" WHERE first_name LIKE 'G%'; // Conditional MAX aggMaxWithHaving diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec index 0f8a16b9e7bb4..4b12d2de58fc7 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec @@ -119,7 +119,10 @@ SELECT DAY_OF_WEEK(birth_date) day, COUNT(*) c FROM test_emp WHERE DAY_OF_WEEK(b currentTimestampYear SELECT YEAR(CURRENT_TIMESTAMP()) AS result; -currentTimestampMonth +// +// H2 uses the local timezone instead of the specified one +// +currentTimestampMonth-Ignore SELECT MONTH(CURRENT_TIMESTAMP()) AS result; currentTimestampHour-Ignore diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec index cfbff2ada573e..af81b060ebd3f 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec @@ -49,6 +49,8 @@ whereFieldWithNotEqualsOnString SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND gender <> 'M'; whereFieldWithLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name LIKE 'K%'; +whereFieldWithNotLikeMatch +SELECT last_name l FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT LIKE 'Ma%'; whereFieldWithOrderNot SELECT last_name l FROM "test_emp" WHERE NOT emp_no < 10003 ORDER BY emp_no LIMIT 5; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index cdc773a91af7a..a67da8d6efd0b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -165,6 +165,7 @@ public static Object nullif(Object left, Object right) { // Regex // public static Boolean regex(String value, String pattern) { + // TODO: this needs to be improved to avoid creating the pattern on every call return RegexOperation.match(value, pattern); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java index a5c8028f6709a..9dc3c69fd2971 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java @@ -11,26 +11,24 @@ public class Like extends RegexMatch { - public Like(Location location, Expression left, LikePattern right) { - super(location, left, right); - } + private final LikePattern pattern; - @Override - protected NodeInfo info() { - return NodeInfo.create(this, Like::new, left(), pattern()); + public Like(Location location, Expression left, LikePattern pattern) { + super(location, left, pattern.asJavaRegex()); + this.pattern = pattern; } public LikePattern pattern() { - return (LikePattern) right(); + return pattern; } @Override - protected Like replaceChildren(Expression newLeft, Expression newRight) { - return new Like(location(), newLeft, (LikePattern) newRight); + protected NodeInfo info() { + return NodeInfo.create(this, Like::new, field(), pattern); } @Override - protected String asString(Expression pattern) { - return ((LikePattern) pattern).asJavaRegex(); + protected Like replaceChild(Expression newLeft) { + return new Like(location(), newLeft, pattern); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java index bde8129f8e72a..d07df617df9f0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java @@ -5,10 +5,6 @@ */ package org.elasticsearch.xpack.sql.expression.predicate.regex; -import org.elasticsearch.xpack.sql.expression.LeafExpression; -import org.elasticsearch.xpack.sql.tree.Location; -import org.elasticsearch.xpack.sql.tree.NodeInfo; -import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.Objects; @@ -21,7 +17,7 @@ * * To prevent conflicts with ES, the string and char must be validated to not contain '*'. */ -public class LikePattern extends LeafExpression { +public class LikePattern { private final String pattern; private final char escape; @@ -30,8 +26,7 @@ public class LikePattern extends LeafExpression { private final String wildcard; private final String indexNameWildcard; - public LikePattern(Location location, String pattern, char escape) { - super(location); + public LikePattern(String pattern, char escape) { this.pattern = pattern; this.escape = escape; // early initialization to force string validation @@ -40,11 +35,6 @@ public LikePattern(Location location, String pattern, char escape) { this.indexNameWildcard = StringUtils.likeToIndexWildcard(pattern, escape); } - @Override - protected NodeInfo info() { - return NodeInfo.create(this, LikePattern::new, pattern, escape); - } - public String pattern() { return pattern; } @@ -74,16 +64,6 @@ public String asIndexNameWildcard() { return indexNameWildcard; } - @Override - public boolean nullable() { - return false; - } - - @Override - public DataType dataType() { - return DataType.KEYWORD; - } - @Override public int hashCode() { return Objects.hash(pattern, escape); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java index 346c3062bfaa8..a09586fd35fb4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java @@ -6,28 +6,29 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.Literal; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; public class RLike extends RegexMatch { - public RLike(Location location, Expression left, Literal right) { - super(location, left, right); + private final String pattern; + + public RLike(Location location, Expression left, String pattern) { + super(location, left, pattern); + this.pattern = pattern; } - @Override - protected NodeInfo info() { - return NodeInfo.create(this, RLike::new, left(), (Literal) right()); + public String pattern() { + return pattern; } @Override - protected RLike replaceChildren(Expression newLeft, Expression newRight) { - return new RLike(location(), newLeft, (Literal) newRight); + protected NodeInfo info() { + return NodeInfo.create(this, RLike::new, field(), pattern); } @Override - protected String asString(Expression pattern) { - return pattern.fold().toString(); + protected RLike replaceChild(Expression newChild) { + return new RLike(location(), newChild, pattern); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java index e1e410064924c..f9390fdfa4514 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java @@ -7,15 +7,19 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.predicate.BinaryPredicate; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; -public abstract class RegexMatch extends BinaryPredicate { +public abstract class RegexMatch extends UnaryScalarFunction { - protected RegexMatch(Location location, Expression value, Expression pattern) { - super(location, value, pattern, RegexOperation.INSTANCE); + private final String pattern; + + protected RegexMatch(Location location, Expression value, String pattern) { + super(location, value); + this.pattern = pattern; } @Override @@ -23,18 +27,25 @@ public DataType dataType() { return DataType.BOOLEAN; } + @Override + public boolean nullable() { + return field().nullable() && pattern != null; + } + @Override public boolean foldable() { // right() is not directly foldable in any context but Like can fold it. - return left().foldable(); + return field().foldable(); } @Override public Boolean fold() { - Object val = left().fold(); - val = val != null ? val.toString() : val; - return function().apply((String) val, asString(right())); + Object val = field().fold(); + return RegexOperation.match(val, pattern); } - protected abstract String asString(Expression pattern); + @Override + protected Processor makeProcessor() { + return new RegexProcessor(pattern); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java deleted file mode 100644 index 7ce8b2b0fc9b4..0000000000000 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.predicate.regex; - -import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.gen.pipeline.BinaryPipe; -import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe; -import org.elasticsearch.xpack.sql.tree.Location; -import org.elasticsearch.xpack.sql.tree.NodeInfo; - -public class RegexPipe extends BinaryPipe { - - public RegexPipe(Location location, Expression expression, Pipe left, Pipe right) { - super(location, expression, left, right); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, RegexPipe::new, expression(), left(), right()); - } - - @Override - protected BinaryPipe replaceChildren(Pipe left, Pipe right) { - return new RegexPipe(location(), expression(), left, right); - } - - @Override - public RegexProcessor asProcessor() { - return new RegexProcessor(left().asProcessor(), right().asProcessor()); - } -} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java index 16f6f0a694966..7f9a2ed76235a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java @@ -7,79 +7,71 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; -import org.elasticsearch.xpack.sql.expression.gen.processor.BinaryProcessor; import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; -import org.elasticsearch.xpack.sql.expression.predicate.PredicateBiFunction; import java.io.IOException; import java.util.Objects; import java.util.regex.Pattern; -public class RegexProcessor extends BinaryProcessor { +public class RegexProcessor implements Processor { - public static class RegexOperation implements PredicateBiFunction { + public static class RegexOperation { - public static final RegexOperation INSTANCE = new RegexOperation(); + public static Boolean match(Object value, Pattern pattern) { + if (pattern == null) { + return Boolean.TRUE; + } - @Override - public String name() { - return symbol(); - } + if (value == null) { + return null; + } - @Override - public String symbol() { - return "REGEX"; + return pattern.matcher(value.toString()).matches(); } - @Override - public Boolean doApply(String value, String pattern) { - return match(value, pattern); - } + public static Boolean match(Object value, String pattern) { + if (pattern == null) { + return Boolean.TRUE; + } - public static Boolean match(Object value, Object pattern) { - if (value == null || pattern == null) { + if (value == null) { return null; } - Pattern p = Pattern.compile(pattern.toString()); - return p.matcher(value.toString()).matches(); + return Pattern.compile(pattern).matcher(value.toString()).matches(); } } public static final String NAME = "rgx"; - public RegexProcessor(Processor value, Processor pattern) { - super(value, pattern); - } + private Pattern pattern; - public RegexProcessor(StreamInput in) throws IOException { - super(in); + public RegexProcessor(String pattern) { + this.pattern = pattern != null ? Pattern.compile(pattern) : null; } @Override - protected Boolean doProcess(Object value, Object pattern) { - return RegexOperation.match(value, pattern); + public String getWriteableName() { + return NAME; } - @Override - protected void checkParameter(Object param) { - if (!(param instanceof String || param instanceof Character)) { - throw new SqlIllegalArgumentException("A string/char is required; received [{}]", param); - } + public RegexProcessor(StreamInput in) throws IOException { + this(in.readOptionalString()); } @Override - public String getWriteableName() { - return NAME; + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(pattern != null ? pattern.toString() : null); } @Override - protected void doWrite(StreamOutput out) throws IOException {} + public Object process(Object input) { + return RegexOperation.match(input, pattern); + } @Override public int hashCode() { - return Objects.hash(left(), right()); + return Objects.hash(pattern); } @Override @@ -93,6 +85,6 @@ public boolean equals(Object obj) { } RegexProcessor other = (RegexProcessor) obj; - return Objects.equals(left(), other.left()) && Objects.equals(right(), other.right()); + return Objects.equals(pattern, other.pattern); } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index cd1cb189b6aa2..f7d659a2933da 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -232,7 +232,7 @@ public Expression visitPredicated(PredicatedContext ctx) { e = new Like(loc, exp, visitPattern(pCtx.pattern())); break; case SqlBaseParser.RLIKE: - e = new RLike(loc, exp, new Literal(source(pCtx.regex), string(pCtx.regex), DataType.KEYWORD)); + e = new RLike(loc, exp, string(pCtx.regex)); break; case SqlBaseParser.NULL: // shortcut to avoid double negation later on (since there's no IsNull (missing in ES is a negated exists)) @@ -301,7 +301,7 @@ public LikePattern visitPattern(PatternContext ctx) { } } - return new LikePattern(source(ctx), pattern, escape); + return new LikePattern(pattern, escape); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index a757bde89e857..af180aae90bdc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeHistogramFunction; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.expression.literal.Intervals; import org.elasticsearch.xpack.sql.expression.predicate.Range; import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate; @@ -103,7 +104,6 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.expression.Foldables.doubleValuesOf; -import static org.elasticsearch.xpack.sql.expression.Foldables.stringValueOf; import static org.elasticsearch.xpack.sql.expression.Foldables.valueOf; final class QueryTranslator { @@ -121,7 +121,8 @@ private QueryTranslator(){} new Likes(), new StringQueries(), new Matches(), - new MultiMatches() + new MultiMatches(), + new Scalars() ); private static final List> AGG_TRANSLATORS = Arrays.asList( @@ -447,13 +448,13 @@ protected QueryTranslation asQuery(RegexMatch e, boolean onAggs) { boolean inexact = true; String target = null; - if (e.left() instanceof FieldAttribute) { - FieldAttribute fa = (FieldAttribute) e.left(); + if (e.field() instanceof FieldAttribute) { + FieldAttribute fa = (FieldAttribute) e.field(); inexact = fa.isInexact(); target = nameOf(inexact ? fa : fa.exactAttribute()); } else { throw new SqlIllegalArgumentException("Scalar function ({}) not allowed (yet) as arguments for LIKE", - Expressions.name(e.left())); + Expressions.name(e.field())); } if (e instanceof Like) { @@ -462,21 +463,21 @@ protected QueryTranslation asQuery(RegexMatch e, boolean onAggs) { q = new QueryStringQuery(e.location(), p.asLuceneWildcard(), target); } else { - q = new WildcardQuery(e.location(), nameOf(e.left()), p.asLuceneWildcard()); + q = new WildcardQuery(e.location(), nameOf(e.field()), p.asLuceneWildcard()); } } if (e instanceof RLike) { - String pattern = stringValueOf(e.right()); + String pattern = ((RLike) e).pattern(); if (inexact) { q = new QueryStringQuery(e.location(), "/" + pattern + "/", target); } else { - q = new RegexQuery(e.location(), nameOf(e.left()), pattern); + q = new RegexQuery(e.location(), nameOf(e.field()), pattern); } } - return q != null ? new QueryTranslation(wrapIfNested(q, e.left())) : null; + return q != null ? new QueryTranslation(wrapIfNested(q, e.field())) : null; } } @@ -529,8 +530,16 @@ protected QueryTranslation asQuery(Not not, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(not.id().toString(), not.asScript()); } else { - query = handleQuery(not, not.field(), - () -> new NotQuery(not.location(), toQuery(not.field(), false).query)); + Expression e = not.field(); + Query wrappedQuery = toQuery(not.field(), false).query; + Query q = wrappedQuery instanceof ScriptQuery ? new ScriptQuery(not.location(), + not.asScript()) : new NotQuery(not.location(), wrappedQuery); + + if (e instanceof FieldAttribute) { + query = wrapIfNested(q, e); + } + + query = q; } return new QueryTranslation(query, aggFilter); @@ -547,8 +556,14 @@ protected QueryTranslation asQuery(IsNotNull isNotNull, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(isNotNull.id().toString(), isNotNull.asScript()); } else { - query = handleQuery(isNotNull, isNotNull.field(), - () -> new ExistsQuery(isNotNull.location(), nameOf(isNotNull.field()))); + Query q = null; + if (isNotNull.field() instanceof FieldAttribute) { + q = new ExistsQuery(isNotNull.location(), nameOf(isNotNull.field())); + } else { + q = new ScriptQuery(isNotNull.location(), isNotNull.asScript()); + } + final Query qu = q; + query = handleQuery(isNotNull, isNotNull.field(), () -> qu); } return new QueryTranslation(query, aggFilter); @@ -565,8 +580,15 @@ protected QueryTranslation asQuery(IsNull isNull, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(isNull.id().toString(), isNull.asScript()); } else { - query = handleQuery(isNull, isNull.field(), - () -> new NotQuery(isNull.location(), new ExistsQuery(isNull.location(), nameOf(isNull.field())))); + Query q = null; + if (isNull.field() instanceof FieldAttribute) { + q = new NotQuery(isNull.location(), new ExistsQuery(isNull.location(), nameOf(isNull.field()))); + } else { + q = new ScriptQuery(isNull.location(), isNull.asScript()); + } + final Query qu = q; + + query = handleQuery(isNull, isNull.field(), () -> qu); } return new QueryTranslation(query, aggFilter); @@ -678,7 +700,14 @@ protected QueryTranslation asQuery(In in, boolean onAggs) { aggFilter = new AggFilter(at.id().toString(), in.asScript()); } else { - query = handleQuery(in, ne, () -> new TermsQuery(in.location(), ne.name(), in.list())); + Query q = null; + if (in.value() instanceof FieldAttribute) { + q = new TermsQuery(in.location(), ne.name(), in.list()); + } else { + q = new ScriptQuery(in.location(), in.asScript()); + } + Query qu = q; + query = handleQuery(in, ne, () -> qu); } return new QueryTranslation(query, aggFilter); } @@ -719,6 +748,25 @@ protected QueryTranslation asQuery(Range r, boolean onAggs) { } } } + + static class Scalars extends ExpressionTranslator { + + @Override + protected QueryTranslation asQuery(ScalarFunction f, boolean onAggs) { + ScriptTemplate script = f.asScript(); + + Query query = null; + AggFilter aggFilter = null; + + if (onAggs) { + aggFilter = new AggFilter(f.id().toString(), script); + } else { + query = handleQuery(f, f, () -> new ScriptQuery(f.location(), script)); + } + + return new QueryTranslation(query, aggFilter); + } + } // @@ -862,8 +910,9 @@ public QueryTranslation translate(Expression exp, boolean onAggs) { protected static Query handleQuery(ScalarFunction sf, Expression field, Supplier query) { + Query q = query.get(); if (field instanceof FieldAttribute) { - return wrapIfNested(query.get(), field); + return wrapIfNested(q, field); } return new ScriptQuery(sf.location(), sf.asScript()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java index 8c8a64c79f298..2412342c69c6e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java @@ -322,10 +322,10 @@ public void testConstantNot() { public void testConstantFoldingLikes() { assertEquals(Literal.TRUE, - new ConstantFolding().rule(new Like(EMPTY, Literal.of(EMPTY, "test_emp"), new LikePattern(EMPTY, "test%", (char) 0))) + new ConstantFolding().rule(new Like(EMPTY, Literal.of(EMPTY, "test_emp"), new LikePattern("test%", (char) 0))) .canonical()); assertEquals(Literal.TRUE, - new ConstantFolding().rule(new RLike(EMPTY, Literal.of(EMPTY, "test_emp"), Literal.of(EMPTY, "test.emp"))).canonical()); + new ConstantFolding().rule(new RLike(EMPTY, Literal.of(EMPTY, "test_emp"), "test.emp")).canonical()); } public void testConstantFoldingDatetime() { @@ -419,7 +419,7 @@ public void testGenericNullableExpression() { // comparison assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), Literal.NULL))); // regex - assertNullLiteral(rule.rule(new RLike(EMPTY, getFieldAttribute(), Literal.NULL))); + assertNullLiteral(rule.rule(new RLike(EMPTY, Literal.NULL, "123"))); } public void testSimplifyCoalesceNulls() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java index 963498bb9b6b2..cc91cdf6eabd7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.expression.predicate.fulltext.FullTextPredicate; import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InPipe; +import org.elasticsearch.xpack.sql.expression.predicate.regex.Like; import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.sql.tree.NodeTests.ChildrenAreAProperty; import org.elasticsearch.xpack.sql.tree.NodeTests.Dummy; @@ -449,14 +450,12 @@ public boolean equals(Object obj) { } return b.toString(); } - } else if (toBuildClass == LikePattern.class) { - /* - * The pattern and escape character have to be valid together - * so we pick an escape character that isn't used - */ - if (argClass == char.class) { - return randomFrom('\\', '|', '/', '`'); + } else if (toBuildClass == Like.class) { + + if (argClass == LikePattern.class) { + return new LikePattern(randomAlphaOfLength(16), randomFrom('\\', '|', '/', '`')); } + } else if (toBuildClass == Histogram.class) { if (argClass == Expression.class) { return LiteralTests.randomLiteral(); From f71b77df7a01077c8740da96faa9f74e9f955ae8 Mon Sep 17 00:00:00 2001 From: Evgenia Badyanova Date: Mon, 17 Dec 2018 11:59:45 -0500 Subject: [PATCH 37/57] Fixing line length for EnvironmentTests and RecoveryTests (#36657) Relates #34884 --- .../resources/checkstyle_suppressions.xml | 6 -- .../elasticsearch/env/EnvironmentTests.java | 3 +- .../env/NodeEnvironmentTests.java | 7 +- .../recovery/FullRollingRestartIT.java | 25 +++++--- .../recovery/RecoveryWhileUnderLoadIT.java | 64 ++++++++++++++----- .../elasticsearch/recovery/RelocationIT.java | 55 +++++++++++----- .../recovery/TruncatedRecoveryIT.java | 6 +- 7 files changed, 113 insertions(+), 53 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 6e628eab0cbd3..55fdcecb084a9 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -64,17 +64,11 @@ - - - - - - diff --git a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java index 5ada31b612941..c87a896d318be 100644 --- a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -55,7 +55,8 @@ public void testRepositoryResolution() throws IOException { Environment environment = newEnvironment(); assertThat(environment.resolveRepoFile("/test/repos/repo1"), nullValue()); assertThat(environment.resolveRepoFile("test/repos/repo1"), nullValue()); - environment = newEnvironment(Settings.builder().putList(Environment.PATH_REPO_SETTING.getKey(), "/test/repos", "/another/repos", "/test/repos/../other").build()); + environment = newEnvironment(Settings.builder() + .putList(Environment.PATH_REPO_SETTING.getKey(), "/test/repos", "/another/repos", "/test/repos/../other").build()); assertThat(environment.resolveRepoFile("/test/repos/repo1"), notNullValue()); assertThat(environment.resolveRepoFile("test/repos/repo1"), notNullValue()); assertThat(environment.resolveRepoFile("/another/repos/repo1"), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 7a24ebaf0484c..63635f5cbe7a4 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -352,7 +352,8 @@ public void run() { for (int i = 0; i < iters; i++) { int shard = randomIntBetween(0, counts.length - 1); try { - try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "fooUUID", shard), scaledRandomIntBetween(0, 10))) { + try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "fooUUID", shard), + scaledRandomIntBetween(0, 10))) { counts[shard].value++; countsAtomic[shard].incrementAndGet(); assertEquals(flipFlop[shard].incrementAndGet(), 1); @@ -386,7 +387,9 @@ public void testCustomDataPaths() throws Exception { final Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "myindexUUID").build(); IndexSettings s1 = IndexSettingsModule.newIndexSettings("myindex", indexSettings); - IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build()); + IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder() + .put(indexSettings) + .put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build()); Index index = new Index("myindex", "myindexUUID"); ShardId sid = new ShardId(index, 0); diff --git a/server/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/server/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java index 6624d4eb8ded4..0fb5f7ac114d6 100644 --- a/server/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java @@ -76,14 +76,16 @@ public void testFullRollingRestart() throws Exception { internalCluster().startNode(settings); // make sure the cluster state is green, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3")); logger.info("--> add two more nodes"); internalCluster().startNode(settings); internalCluster().startNode(settings); // make sure the cluster state is green, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("5")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("5")); logger.info("--> refreshing and checking data"); refresh(); @@ -94,11 +96,13 @@ public void testFullRollingRestart() throws Exception { // now start shutting nodes down internalCluster().stopRandomDataNode(); // make sure the cluster state is green, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("4")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("4")); internalCluster().stopRandomDataNode(); // make sure the cluster state is green, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("3")); logger.info("--> stopped two nodes, verifying data"); refresh(); @@ -109,12 +113,14 @@ public void testFullRollingRestart() throws Exception { // closing the 3rd node internalCluster().stopRandomDataNode(); // make sure the cluster state is green, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("2")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForGreenStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("2")); internalCluster().stopRandomDataNode(); // make sure the cluster state is yellow, and all has been recovered - assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout).setWaitForYellowStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("1")); + assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout(healthTimeout) + .setWaitForYellowStatus().setWaitForNoRelocatingShards(true).setWaitForNodes("1")); logger.info("--> one node left, verifying data"); refresh(); @@ -133,7 +139,9 @@ public void testNoRebalanceOnRollingRestart() throws Exception { * to relocating to the restarting node since all had 2 shards and now one node has nothing allocated. * We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen. */ - prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "6").put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMinutes(1))).get(); + prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "6") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMinutes(1))).get(); for (int i = 0; i < 100; i++) { client().prepareIndex("test", "type1", Long.toString(i)) @@ -152,7 +160,8 @@ public void testNoRebalanceOnRollingRestart() throws Exception { recoveryResponse = client().admin().indices().prepareRecoveries("test").get(); for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) { - assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state, + assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state, recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false); } } diff --git a/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index 0d2235c30a425..c0345be6fae01 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -53,14 +53,18 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -@TestLogging("_root:DEBUG,org.elasticsearch.index.shard:TRACE,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.index.seqno:TRACE,org.elasticsearch.indices.recovery:TRACE") +@TestLogging("_root:DEBUG,org.elasticsearch.index.shard:TRACE,org.elasticsearch.cluster.service:TRACE," + + "org.elasticsearch.index.seqno:TRACE,org.elasticsearch.indices.recovery:TRACE") public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { private final Logger logger = LogManager.getLogger(RecoveryWhileUnderLoadIT.class); public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); - assertAcked(prepareCreate("test", 1, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); + assertAcked(prepareCreate("test", 1, Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); final int totalNumDocs = scaledRandomIntBetween(200, 10000); int waitFor = totalNumDocs / 10; @@ -92,7 +96,8 @@ public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { logger.info("--> waiting for GREEN health status ..."); // make sure the cluster state is green, and all has been recovered - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); waitForDocs(totalNumDocs, indexer); @@ -113,7 +118,10 @@ public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); - assertAcked(prepareCreate("test", 1, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); + assertAcked(prepareCreate("test", 1, Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); final int totalNumDocs = scaledRandomIntBetween(200, 10000); int waitFor = totalNumDocs / 10; @@ -142,7 +150,8 @@ public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() thr allowNodes("test", 4); logger.info("--> waiting for GREEN health status ..."); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); @@ -164,7 +173,9 @@ public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() thr public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); - assertAcked(prepareCreate("test", 2, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); + assertAcked(prepareCreate("test", 2, Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); final int totalNumDocs = scaledRandomIntBetween(200, 10000); int waitFor = totalNumDocs / 10; @@ -194,7 +205,10 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception allowNodes("test", 4); logger.info("--> waiting for GREEN health status ..."); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNoRelocatingShards(true)); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m") + .setWaitForGreenStatus() + .setWaitForNoRelocatingShards(true)); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); waitForDocs(totalNumDocs, indexer); @@ -205,23 +219,31 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception logger.info("--> allow 3 nodes for index [test] ..."); allowNodes("test", 3); logger.info("--> waiting for relocations ..."); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m") + .setWaitForNoRelocatingShards(true)); logger.info("--> allow 2 nodes for index [test] ..."); allowNodes("test", 2); logger.info("--> waiting for relocations ..."); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m") + .setWaitForNoRelocatingShards(true)); logger.info("--> allow 1 nodes for index [test] ..."); allowNodes("test", 1); logger.info("--> waiting for relocations ..."); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m") + .setWaitForNoRelocatingShards(true)); logger.info("--> marking and waiting for indexing threads to stop ..."); indexer.stop(); logger.info("--> indexing threads stopped"); - assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)); + assertNoTimeout(client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID).setTimeout("5m") + .setWaitForNoRelocatingShards(true)); logger.info("--> refreshing the index"); refreshAndAssert(); @@ -235,7 +257,10 @@ public void testRecoverWhileRelocating() throws Exception { final int numReplicas = 0; logger.info("--> creating test index ..."); int allowNodes = 2; - assertAcked(prepareCreate("test", 3, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); + assertAcked(prepareCreate("test", 3, Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, numShards) + .put(SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC))); final int numDocs = scaledRandomIntBetween(200, 9999); @@ -258,7 +283,8 @@ public void testRecoverWhileRelocating() throws Exception { logger.info("--> indexing threads stopped"); logger.info("--> bump up number of replicas to 1 and allow all nodes to hold the index"); allowNodes("test", 3); - assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("number_of_replicas", 1)).get()); + assertAcked(client().admin().indices().prepareUpdateSettings("test") + .setSettings(Settings.builder().put("number_of_replicas", 1)).get()); ensureGreen(TimeValue.timeValueMinutes(5)); logger.info("--> refreshing the index"); @@ -273,7 +299,8 @@ private void iterateAssertCount(final int numberOfShards, final int iterations, SearchResponse[] iterationResults = new SearchResponse[iterations]; boolean error = false; for (int i = 0; i < iterations; i++) { - SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get(); + SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()) + .addSort("id", SortOrder.ASC).get(); logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse); iterationResults[i] = searchResponse; if (searchResponse.getHits().getTotalHits().value != numberOfDocs) { @@ -286,7 +313,8 @@ private void iterateAssertCount(final int numberOfShards, final int iterations, IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().get(); for (ShardStats shardStats : indicesStatsResponse.getShards()) { DocsStats docsStats = shardStats.getStats().docs; - logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary()); + logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), + shardStats.getShardRouting().primary()); } ClusterService clusterService = clusterService(); @@ -332,12 +360,14 @@ private void iterateAssertCount(final int numberOfShards, final int iterations, } private void logSearchResponse(int numberOfShards, long numberOfDocs, int iteration, SearchResponse searchResponse) { - logger.info("iteration [{}] - successful shards: {} (expected {})", iteration, searchResponse.getSuccessfulShards(), numberOfShards); + logger.info("iteration [{}] - successful shards: {} (expected {})", iteration, + searchResponse.getSuccessfulShards(), numberOfShards); logger.info("iteration [{}] - failed shards: {} (expected 0)", iteration, searchResponse.getFailedShards()); if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { logger.info("iteration [{}] - shard failures: {}", iteration, Arrays.toString(searchResponse.getShardFailures())); } - logger.info("iteration [{}] - returned documents: {} (expected {})", iteration, searchResponse.getHits().getTotalHits().value, numberOfDocs); + logger.info("iteration [{}] - returned documents: {} (expected {})", iteration, + searchResponse.getHits().getTotalHits().value, numberOfDocs); } private void refreshAndAssert() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index b27e4fd229a07..62208a404885b 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -133,7 +133,8 @@ public void testSimpleRelocationNoIndexing() { logger.info("--> start another node"); final String node_2 = internalCluster().startNode(); - ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").execute().actionGet(); + ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) + .setWaitForNodes("2").execute().actionGet(); assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); logger.info("--> relocate the shard from node1 to node2"); @@ -141,7 +142,8 @@ public void testSimpleRelocationNoIndexing() { .add(new MoveAllocationCommand("test", 0, node_1, node_2)) .execute().actionGet(); - clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet(); + clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet(); assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); logger.info("--> verifying count again..."); @@ -155,7 +157,8 @@ public void testRelocationWhileIndexingRandom() throws Exception { int numberOfReplicas = randomBoolean() ? 0 : 1; int numberOfNodes = numberOfReplicas == 0 ? 2 : 3; - logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", numberOfRelocations, numberOfReplicas, numberOfNodes); + logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", + numberOfRelocations, numberOfReplicas, numberOfNodes); String[] nodes = new String[numberOfNodes]; logger.info("--> starting [node1] ..."); @@ -172,8 +175,10 @@ public void testRelocationWhileIndexingRandom() throws Exception { logger.info("--> starting [node{}] ...", i); nodes[i - 1] = internalCluster().startNode(); if (i != numberOfNodes) { - ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) - .setWaitForNodes(Integer.toString(i)).setWaitForGreenStatus().execute().actionGet(); + ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNodes(Integer.toString(i)) + .setWaitForGreenStatus().execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); } } @@ -202,7 +207,10 @@ public void testRelocationWhileIndexingRandom() throws Exception { logger.debug("--> flushing"); client().admin().indices().prepareFlush().get(); } - ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet(); + ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true) + .setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet(); assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); indexer.pauseIndexing(); logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode); @@ -218,7 +226,8 @@ public void testRelocationWhileIndexingRandom() throws Exception { boolean ranOnce = false; for (int i = 0; i < 10; i++) { logger.info("--> START search test round {}", i + 1); - SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).storedFields().execute().actionGet().getHits(); + SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()) + .setSize((int) indexer.totalIndexedDocs()).storedFields().execute().actionGet().getHits(); ranOnce = true; if (hits.getTotalHits().value != indexer.totalIndexedDocs()) { int[] hitIds = new int[(int) indexer.totalIndexedDocs()]; @@ -252,7 +261,8 @@ public void testRelocationWhileRefreshing() throws Exception { int numberOfReplicas = randomBoolean() ? 0 : 1; int numberOfNodes = numberOfReplicas == 0 ? 2 : 3; - logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", numberOfRelocations, numberOfReplicas, numberOfNodes); + logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", + numberOfRelocations, numberOfReplicas, numberOfNodes); String[] nodes = new String[numberOfNodes]; logger.info("--> starting [node_0] ..."); @@ -281,13 +291,15 @@ public void testRelocationWhileRefreshing() throws Exception { final Semaphore postRecoveryShards = new Semaphore(0); final IndexEventListener listener = new IndexEventListener() { @Override - public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { + public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, + IndexShardState currentState, @Nullable String reason) { if (currentState == IndexShardState.POST_RECOVERY) { postRecoveryShards.release(); } } }; - for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getInstances(MockIndexEventListener.TestEventListener.class)) { + for (MockIndexEventListener.TestEventListener eventListener : internalCluster() + .getInstances(MockIndexEventListener.TestEventListener.class)) { eventListener.setNewDelegate(listener); } @@ -327,7 +339,10 @@ public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardSt indexRandom(true, true, builders2); // verify cluster was finished. - assertFalse(client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).setWaitForEvents(Priority.LANGUID).setTimeout("30s").get().isTimedOut()); + assertFalse(client().admin().cluster().prepareHealth() + .setWaitForNoRelocatingShards(true) + .setWaitForEvents(Priority.LANGUID) + .setTimeout("30s").get().isTimedOut()); logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode); logger.debug("--> verifying all searches return the same number of docs"); @@ -374,17 +389,20 @@ public void testCancellationCleansTempFiles() throws Exception { MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, p_node); for (DiscoveryNode node : clusterService.state().nodes()) { if (!node.equals(clusterService.localNode())) { - mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(corruptionCount)); + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, node.getName()), + new RecoveryCorruption(corruptionCount)); } } - client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)).get(); + client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)).get(); corruptionCount.await(); logger.info("--> stopping replica assignment"); assertAcked(client().admin().cluster().prepareUpdateSettings() - .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"))); + .setTransientSettings(Settings.builder() + .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"))); logger.info("--> wait for all replica shards to be removed, on all nodes"); assertBusy(() -> { @@ -408,7 +426,8 @@ public void testCancellationCleansTempFiles() throws Exception { Files.walkFileTree(shardLoc, new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - assertThat("found a temporary recovery file: " + file, file.getFileName().toString(), not(startsWith("recovery."))); + assertThat("found a temporary recovery file: " + file, file.getFileName().toString(), + not(startsWith("recovery."))); return FileVisitResult.CONTINUE; } }); @@ -496,13 +515,15 @@ class RecoveryCorruption implements StubbableTransport.SendRequestBehavior { } @Override - public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { + public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest chunkRequest = (RecoveryFileChunkRequest) request; if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) { // corrupting the segments_N files in order to make sure future recovery re-send files logger.debug("corrupting [{}] to {}. file name: [{}]", action, connection.getNode(), chunkRequest.name()); - assert chunkRequest.content().toBytesRef().bytes == chunkRequest.content().toBytesRef().bytes : "no internal reference!!"; + assert chunkRequest.content().toBytesRef().bytes == + chunkRequest.content().toBytesRef().bytes : "no internal reference!!"; byte[] array = chunkRequest.content().toBytesRef().bytes; array[0] = (byte) ~array[0]; // flip one byte in the content corruptionCount.countDown(); diff --git a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index ac8688c9847d3..973c687ebe84c 100644 --- a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -89,7 +89,8 @@ public void testCancelRecoveryAndResume() throws Exception { // we have no replicas so far and make sure that we allocate the primary on the lucky node assertAcked(prepareCreate("test") .addMapping("type1", "field1", "type=text", "the_id", "type=text") - .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards()) + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards()) .put("index.routing.allocation.include._name", primariesNode.getNode().getName()))); // only allocate on the lucky node // index some docs and check if they are coming back @@ -112,7 +113,8 @@ public void testCancelRecoveryAndResume() throws Exception { final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean truncate = new AtomicBoolean(true); for (NodeStats dataNode : dataNodeStats) { - MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName())); + MockTransportService mockTransportService = ((MockTransportService) internalCluster() + .getInstance(TransportService.class, dataNode.getNode().getName())); mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { From e9bd724b21e4d972243e6d06abcc490e70e1b315 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 17 Dec 2018 19:06:23 +0200 Subject: [PATCH 38/57] Add back one line removed by mistake regarding java version check and COMPAT jvm parameter existence --- .../function/scalar/datetime/NamedDateTimeProcessorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java index ae152bba5d820..3531152c69b8b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java @@ -41,6 +41,7 @@ protected NamedDateTimeProcessor mutateInstance(NamedDateTimeProcessor instance) } public void testValidDayNamesInUTC() { + assumeJava9PlusAndCompatLocaleProviderSetting(); NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, UTC); assertEquals("Thursday", proc.process(dateTime(0L))); assertEquals("Saturday", proc.process(dateTime(-64164233612338L))); From 86e6d18c224e52257c8c0a02400817b5b47d5a08 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 17 Dec 2018 13:24:31 -0500 Subject: [PATCH 39/57] Do not resolve addresses in remote connection info (#36671) The remote connection info API leads to resolving addresses of seed nodes when invoked. This is problematic because if a hostname fails to resolve, we would not display any remote connection info. Yet, a hostname not resolving can happen across remote clusters, especially in the modern world of cloud services with dynamically chaning IPs. Instead, the remote connection info API should be providing the configured seed nodes. This commit changes the remote connection info to display the configured seed nodes, avoiding a hostname resolution. Note that care was taken to preserve backwards compatibility with previous versions that expect the remote connection info to serialize a transport address instead of a string representing the hostname. --- .../transport/RemoteClusterAware.java | 29 ++-- .../transport/RemoteClusterConnection.java | 25 ++- .../transport/RemoteClusterService.java | 16 +- .../transport/RemoteConnectionInfo.java | 52 +++++- .../RemoteClusterConnectionTests.java | 163 +++++++++--------- .../transport/RemoteClusterServiceTests.java | 27 +-- 6 files changed, 188 insertions(+), 124 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 2c36af8638f63..237e73e572ae3 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -183,10 +183,11 @@ protected RemoteClusterAware(Settings settings) { * (ProxyAddresss, [SeedNodeSuppliers]). If a cluster is configured with a proxy address all seed nodes will point to * {@link TransportAddress#META_ADDRESS} and their configured address will be used as the hostname for the generated discovery node. */ - protected static Map>>> buildRemoteClustersDynamicConfig(Settings settings) { - final Map>>> remoteSeeds = + protected static Map>>>> buildRemoteClustersDynamicConfig( + final Settings settings) { + final Map>>>> remoteSeeds = buildRemoteClustersDynamicConfig(settings, REMOTE_CLUSTERS_SEEDS); - final Map>>> searchRemoteSeeds = + final Map>>>> searchRemoteSeeds = buildRemoteClustersDynamicConfig(settings, SEARCH_REMOTE_CLUSTERS_SEEDS); // sort the intersection for predictable output order final NavigableSet intersection = @@ -205,7 +206,7 @@ protected static Map>>> build .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private static Map>>> buildRemoteClustersDynamicConfig( + private static Map>>>> buildRemoteClustersDynamicConfig( final Settings settings, final Setting.AffixSetting> seedsSetting) { final Stream>> allConcreteSettings = seedsSetting.getAllConcreteSettings(settings); return allConcreteSettings.collect( @@ -214,9 +215,9 @@ private static Map>>> buildRe List addresses = concreteSetting.get(settings); final boolean proxyMode = REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).existsOrFallbackExists(settings); - List> nodes = new ArrayList<>(addresses.size()); + List>> nodes = new ArrayList<>(addresses.size()); for (String address : addresses) { - nodes.add(() -> buildSeedNode(clusterName, address, proxyMode)); + nodes.add(Tuple.tuple(address, () -> buildSeedNode(clusterName, address, proxyMode))); } return new Tuple<>(REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).get(settings), nodes); })); @@ -304,16 +305,24 @@ public void listenForUpdates(ClusterSettings clusterSettings) { (namespace, value) -> {}); } - - protected static InetSocketAddress parseSeedAddress(String remoteHost) { - String host = remoteHost.substring(0, indexOfPortSeparator(remoteHost)); + static InetSocketAddress parseSeedAddress(String remoteHost) { + final Tuple hostPort = parseHostPort(remoteHost); + final String host = hostPort.v1(); + assert hostPort.v2() != null : remoteHost; + final int port = hostPort.v2(); InetAddress hostAddress; try { hostAddress = InetAddress.getByName(host); } catch (UnknownHostException e) { throw new IllegalArgumentException("unknown host [" + host + "]", e); } - return new InetSocketAddress(hostAddress, parsePort(remoteHost)); + return new InetSocketAddress(hostAddress, port); + } + + public static Tuple parseHostPort(final String remoteHost) { + final String host = remoteHost.substring(0, indexOfPortSeparator(remoteHost)); + final int port = parsePort(remoteHost); + return Tuple.tuple(host, port); } private static int parsePort(String remoteHost) { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 87dd99e6590f2..7ea55925262ff 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -35,6 +35,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -95,7 +96,7 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos private final Predicate nodePredicate; private final ThreadPool threadPool; private volatile String proxyAddress; - private volatile List> seedNodes; + private volatile List>> seedNodes; private volatile boolean skipUnavailable; private final ConnectHandler connectHandler; private final TimeValue initialConnectionTimeout; @@ -111,7 +112,7 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos * @param nodePredicate a predicate to filter eligible remote nodes to connect to * @param proxyAddress the proxy address */ - RemoteClusterConnection(Settings settings, String clusterAlias, List> seedNodes, + RemoteClusterConnection(Settings settings, String clusterAlias, List>> seedNodes, TransportService transportService, int maxNumRemoteConnections, Predicate nodePredicate, String proxyAddress) { this(settings, clusterAlias, seedNodes, transportService, maxNumRemoteConnections, nodePredicate, proxyAddress, @@ -119,7 +120,7 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos } // Public for tests to pass a StubbableConnectionManager - RemoteClusterConnection(Settings settings, String clusterAlias, List> seedNodes, + RemoteClusterConnection(Settings settings, String clusterAlias, List>> seedNodes, TransportService transportService, int maxNumRemoteConnections, Predicate nodePredicate, String proxyAddress, ConnectionManager connectionManager) { this.transportService = transportService; @@ -155,7 +156,10 @@ private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, Discovery /** * Updates the list of seed nodes for this cluster connection */ - synchronized void updateSeedNodes(String proxyAddress, List> seedNodes, ActionListener connectListener) { + synchronized void updateSeedNodes( + final String proxyAddress, + final List>> seedNodes, + final ActionListener connectListener) { this.seedNodes = Collections.unmodifiableList(new ArrayList<>(seedNodes)); this.proxyAddress = proxyAddress; connectHandler.connect(connectListener); @@ -465,7 +469,7 @@ protected void doRun() { maybeConnect(); } }); - collectRemoteNodes(seedNodes.iterator(), transportService, connectionManager, listener); + collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), transportService, connectionManager, listener); } }); } @@ -672,10 +676,13 @@ void addConnectedNode(DiscoveryNode node) { * Get the information about remote nodes to be rendered on {@code _remote/info} requests. */ public RemoteConnectionInfo getConnectionInfo() { - List seedNodeAddresses = seedNodes.stream().map(node -> node.get().getAddress()).collect - (Collectors.toList()); - return new RemoteConnectionInfo(clusterAlias, seedNodeAddresses, maxNumRemoteConnections, connectedNodes.size(), - initialConnectionTimeout, skipUnavailable); + return new RemoteConnectionInfo( + clusterAlias, + seedNodes.stream().map(Tuple::v1).collect(Collectors.toList()), + maxNumRemoteConnections, + connectedNodes.size(), + initialConnectionTimeout, + skipUnavailable); } int getNumNodesConnected() { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index fda0b90f19ea5..cb802f13fdb50 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -201,7 +201,7 @@ public String getKey(final String key) { * @param seeds a cluster alias to discovery node mapping representing the remote clusters seeds nodes * @param connectionListener a listener invoked once every configured cluster has been connected to */ - private synchronized void updateRemoteClusters(Map>>> seeds, + private synchronized void updateRemoteClusters(Map>>>> seeds, ActionListener connectionListener) { if (seeds.containsKey(LOCAL_CLUSTER_GROUP_KEY)) { throw new IllegalArgumentException("remote clusters must not have the empty string as its key"); @@ -212,8 +212,8 @@ private synchronized void updateRemoteClusters(Map>>> entry : seeds.entrySet()) { - List> seedList = entry.getValue().v2(); + for (Map.Entry>>>> entry : seeds.entrySet()) { + List>> seedList = entry.getValue().v2(); String proxyAddress = entry.getValue().v1(); RemoteClusterConnection remote = this.remoteClusters.get(entry.getKey()); @@ -408,9 +408,10 @@ void updateRemoteCluster( final List addresses, final String proxyAddress, final ActionListener connectionListener) { - final List> nodes = addresses.stream().>map(address -> () -> - buildSeedNode(clusterAlias, address, Strings.hasLength(proxyAddress)) - ).collect(Collectors.toList()); + final List>> nodes = + addresses.stream().>>map(address -> Tuple.tuple(address, () -> + buildSeedNode(clusterAlias, address, Strings.hasLength(proxyAddress))) + ).collect(Collectors.toList()); updateRemoteClusters(Collections.singletonMap(clusterAlias, new Tuple<>(proxyAddress, nodes)), connectionListener); } @@ -421,7 +422,8 @@ void updateRemoteCluster( void initializeRemoteClusters() { final TimeValue timeValue = REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings); final PlainActionFuture future = new PlainActionFuture<>(); - Map>>> seeds = RemoteClusterAware.buildRemoteClustersDynamicConfig(settings); + Map>>>> seeds = + RemoteClusterAware.buildRemoteClustersDynamicConfig(settings); updateRemoteClusters(seeds, future); try { future.get(timeValue.millis(), TimeUnit.MILLISECONDS); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java index c2024e39228aa..7c51ca7b9c892 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport; import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -27,25 +29,29 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; -import static java.util.Collections.emptyList; - import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; /** * This class encapsulates all remote cluster information to be rendered on * {@code _remote/info} requests. */ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable { - final List seedNodes; + final List seedNodes; final int connectionsPerCluster; final TimeValue initialConnectionTimeout; final int numNodesConnected; final String clusterAlias; final boolean skipUnavailable; - RemoteConnectionInfo(String clusterAlias, List seedNodes, + RemoteConnectionInfo(String clusterAlias, List seedNodes, int connectionsPerCluster, int numNodesConnected, TimeValue initialConnectionTimeout, boolean skipUnavailable) { this.clusterAlias = clusterAlias; @@ -57,7 +63,17 @@ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable } public RemoteConnectionInfo(StreamInput input) throws IOException { - seedNodes = input.readList(TransportAddress::new); + if (input.getVersion().onOrAfter(Version.V_7_0_0)) { + seedNodes = Arrays.asList(input.readStringArray()); + } else { + // versions prior to 7.0.0 sent the resolved transport address of the seed nodes + final List transportAddresses = input.readList(TransportAddress::new); + seedNodes = + transportAddresses + .stream() + .map(a -> a.address().getHostString() + ":" + a.address().getPort()) + .collect(Collectors.toList()); + } if (input.getVersion().before(Version.V_7_0_0)) { /* * Versions before 7.0 sent the HTTP addresses of all nodes in the @@ -78,7 +94,26 @@ public RemoteConnectionInfo(StreamInput input) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeList(seedNodes); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeStringArray(seedNodes.toArray(new String[0])); + } else { + // versions prior to 7.0.0 received the resolved transport address of the seed nodes + out.writeList(seedNodes + .stream() + .map( + s -> { + final Tuple hostPort = RemoteClusterAware.parseHostPort(s); + assert hostPort.v2() != null : s; + try { + return new TransportAddress( + InetAddress.getByAddress(hostPort.v1(), TransportAddress.META_ADDRESS.getAddress()), + hostPort.v2()); + } catch (final UnknownHostException e) { + throw new AssertionError(e); + } + }) + .collect(Collectors.toList())); + } if (out.getVersion().before(Version.V_7_0_0)) { /* * Versions before 7.0 sent the HTTP addresses of all nodes in the @@ -104,8 +139,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(clusterAlias); { builder.startArray("seeds"); - for (TransportAddress addr : seedNodes) { - builder.value(addr.toString()); + for (String addr : seedNodes) { + builder.value(addr); } builder.endArray(); builder.field("connected", numNodesConnected > 0); @@ -136,4 +171,5 @@ public int hashCode() { return Objects.hash(seedNodes, connectionsPerCluster, initialConnectionTimeout, numNodesConnected, clusterAlias, skipUnavailable); } + } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 50391162367b7..02e701ed4bc86 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; @@ -80,6 +81,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -164,9 +166,9 @@ public void testRemoteProfileIsUsedForLocalCluster() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); assertTrue(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); @@ -206,9 +208,9 @@ public void testRemoteProfileIsUsedForRemoteCluster() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); assertTrue(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); @@ -259,9 +261,9 @@ public void testDiscoverSingleNode() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); assertTrue(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); @@ -282,7 +284,9 @@ public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { knownNodes.add(discoverableTransport.getLocalDiscoNode()); knownNodes.add(incompatibleTransport.getLocalDiscoNode()); Collections.shuffle(knownNodes, random()); - List> seedNodes = Arrays.asList(() -> incompatibleSeedNode, () -> seedNode); + List>> seedNodes = Arrays.asList( + Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode), + Tuple.tuple(seedNode.toString(), () -> seedNode)); Collections.shuffle(seedNodes, random()); try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -317,9 +321,9 @@ public void testNodeDisconnected() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); assertTrue(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); assertFalse(connectionManager.nodeConnected(spareNode)); @@ -367,9 +371,9 @@ public void testFilterDiscoveredNodes() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); if (rejectedNode.equals(seedNode)) { assertFalse(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); @@ -382,11 +386,15 @@ public void testFilterDiscoveredNodes() throws Exception { } } } - private void updateSeedNodes(RemoteClusterConnection connection, List> seedNodes) throws Exception { + private void updateSeedNodes( + final RemoteClusterConnection connection, final List>> seedNodes) throws Exception { updateSeedNodes(connection, seedNodes, null); } - private void updateSeedNodes(RemoteClusterConnection connection, List> seedNodes, String proxyAddress) + private void updateSeedNodes( + final RemoteClusterConnection connection, + final List>> seedNodes, + final String proxyAddress) throws Exception { CountDownLatch latch = new CountDownLatch(1); AtomicReference exceptionAtomicReference = new AtomicReference<>(); @@ -428,9 +436,11 @@ public void testConnectWithIncompatibleTransports() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - expectThrows(Exception.class, () -> updateSeedNodes(connection, Arrays.asList(() -> seedNode))); + expectThrows( + Exception.class, + () -> updateSeedNodes(connection, Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)))); assertFalse(connectionManager.nodeConnected(seedNode)); assertTrue(connection.assertNoRunningConnections()); } @@ -481,7 +491,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { connection.addConnectedNode(seedNode); for (DiscoveryNode node : knownNodes) { final Transport.Connection transportConnection = connection.getConnection(node); @@ -524,7 +534,7 @@ public void run() { CountDownLatch listenerCalled = new CountDownLatch(1); AtomicReference exceptionReference = new AtomicReference<>(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { ActionListener listener = ActionListener.wrap(x -> { listenerCalled.countDown(); fail("expected exception"); @@ -532,7 +542,7 @@ public void run() { exceptionReference.set(x); listenerCalled.countDown(); }); - connection.updateSeedNodes(null, Arrays.asList(() -> seedNode), listener); + connection.updateSeedNodes(null, seedNodes(seedNode), listener); acceptedLatch.await(); connection.close(); // now close it, this should trigger an interrupt on the socket and we can move on assertTrue(connection.assertNoRunningConnections()); @@ -548,6 +558,18 @@ public void run() { } } + private List>> seedNodes(final DiscoveryNode... seedNodes) { + if (seedNodes.length == 0) { + return Collections.emptyList(); + } else if (seedNodes.length == 1) { + return Collections.singletonList(Tuple.tuple(seedNodes[0].toString(), () -> seedNodes[0])); + } else { + return Arrays.stream(seedNodes) + .map(s -> Tuple.tuple(s.toString(), (Supplier)() -> s)) + .collect(Collectors.toList()); + } + } + public void testFetchShards() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); @@ -559,11 +581,11 @@ public void testFetchShards() throws Exception { try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { service.start(); service.acceptIncomingRequests(); - List> nodes = Collections.singletonList(() -> seedNode); + final List>> seedNodes = seedNodes(seedNode); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - nodes, service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes, service, Integer.MAX_VALUE, n -> true, null)) { if (randomBoolean()) { - updateSeedNodes(connection, nodes); + updateSeedNodes(connection, seedNodes); } if (randomBoolean()) { connection.updateSkipUnavailable(randomBoolean()); @@ -599,9 +621,9 @@ public void testFetchShardsThreadContextHeader() throws Exception { try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { service.start(); service.acceptIncomingRequests(); - List> nodes = Collections.singletonList(() -> seedNode); + final List>> seedNodes = seedNodes(seedNode); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - nodes, service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes, service, Integer.MAX_VALUE, n -> true, null)) { SearchRequest request = new SearchRequest("test-index"); Thread[] threads = new Thread[10]; for (int i = 0; i < threads.length; i++) { @@ -655,7 +677,7 @@ public void testFetchShardsSkipUnavailable() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Collections.singletonList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); SearchRequest request = new SearchRequest("test-index"); @@ -759,7 +781,7 @@ public void testTriggerUpdatesConcurrently() throws IOException, InterruptedExce knownNodes.add(discoverableTransport.getLocalDiscoNode()); knownNodes.add(seedTransport1.getLocalDiscoNode()); Collections.shuffle(knownNodes, random()); - List> seedNodes = Arrays.asList(() -> seedNode1, () -> seedNode); + List>> seedNodes = seedNodes(seedNode1, seedNode); Collections.shuffle(seedNodes, random()); try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -839,7 +861,7 @@ public void testCloseWhileConcurrentlyConnecting() throws IOException, Interrupt knownNodes.add(discoverableTransport.getLocalDiscoNode()); knownNodes.add(seedTransport1.getLocalDiscoNode()); Collections.shuffle(knownNodes, random()); - List> seedNodes = Arrays.asList(() -> seedNode1, () -> seedNode); + List>> seedNodes = seedNodes(seedNode1, seedNode); Collections.shuffle(seedNodes, random()); try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -926,7 +948,7 @@ public void testGetConnectionInfo() throws Exception { knownNodes.add(transport3.getLocalDiscoNode()); knownNodes.add(transport2.getLocalDiscoNode()); Collections.shuffle(knownNodes, random()); - List> seedNodes = Arrays.asList(() -> node3, () -> node1, () -> node2); + List>> seedNodes = seedNodes(node3, node1, node2); Collections.shuffle(seedNodes, random()); try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -958,44 +980,32 @@ public void testGetConnectionInfo() throws Exception { } public void testRemoteConnectionInfo() throws IOException { - RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 3, TimeValue.timeValueMinutes(30), false); + RemoteConnectionInfo stats = + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats); - RemoteConnectionInfo stats1 = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 4, TimeValue.timeValueMinutes(30), true); + RemoteConnectionInfo stats1 = + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 4, TimeValue.timeValueMinutes(30), true); assertSerialization(stats1); assertNotEquals(stats, stats1); - stats1 = new RemoteConnectionInfo("test_cluster_1", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 3, TimeValue.timeValueMinutes(30), false); + stats1 = new RemoteConnectionInfo("test_cluster_1", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); - stats1 = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 15)), - 4, 3, TimeValue.timeValueMinutes(30), false); + stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:15"), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); - stats1 = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 3, TimeValue.timeValueMinutes(30), true); + stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), true); assertSerialization(stats1); assertNotEquals(stats, stats1); - stats1 = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 3, TimeValue.timeValueMinutes(325), true); + stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(325), true); assertSerialization(stats1); assertNotEquals(stats, stats1); - stats1 = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 5, 3, TimeValue.timeValueMinutes(30), false); + stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 5, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); } @@ -1016,9 +1026,8 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf public void testRemoteConnectionInfoBwComp() throws IOException { final Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); - RemoteConnectionInfo expected = new RemoteConnectionInfo("test_cluster", - Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 4, new TimeValue(30, TimeUnit.MINUTES), false); + RemoteConnectionInfo expected = + new RemoteConnectionInfo("test_cluster", Arrays.asList("0.0.0.0:1"), 4, 4, new TimeValue(30, TimeUnit.MINUTES), false); // This version was created using the serialization code in use from 6.1 but before 7.0 String encoded = "AQQAAAAABzAuMC4wLjAAAAABAQQAAAAABzAuMC4wLjAAAABQBDwEBAx0ZXN0X2NsdXN0ZXIA"; @@ -1042,27 +1051,25 @@ public void testRemoteConnectionInfoBwComp() throws IOException { } public void testRenderConnectionInfoXContent() throws IOException { - RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 3, TimeValue.timeValueMinutes(30), true); + RemoteConnectionInfo stats = + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), true); stats = assertSerialization(stats); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); stats.toXContent(builder, null); builder.endObject(); - assertEquals("{\"test_cluster\":{\"seeds\":[\"0.0.0.0:1\"],\"connected\":true," + + assertEquals("{\"test_cluster\":{\"seeds\":[\"seed:1\"],\"connected\":true," + "\"num_nodes_connected\":3,\"max_connections_per_cluster\":4,\"initial_connect_timeout\":\"30m\"," + "\"skip_unavailable\":true}}", Strings.toString(builder)); - stats = new RemoteConnectionInfo("some_other_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1), new TransportAddress(TransportAddress.META_ADDRESS, 2)), - 2, 0, TimeValue.timeValueSeconds(30), false); + stats = new RemoteConnectionInfo( + "some_other_cluster", Arrays.asList("seed:1", "seed:2"), 2, 0, TimeValue.timeValueSeconds(30), false); stats = assertSerialization(stats); builder = XContentFactory.jsonBuilder(); builder.startObject(); stats.toXContent(builder, null); builder.endObject(); - assertEquals("{\"some_other_cluster\":{\"seeds\":[\"0.0.0.0:1\",\"0.0.0.0:2\"]," + assertEquals("{\"some_other_cluster\":{\"seeds\":[\"seed:1\",\"seed:2\"]," + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," + "\"skip_unavailable\":false}}", Strings.toString(builder)); } @@ -1081,7 +1088,7 @@ public void testEnsureConnected() throws IOException, InterruptedException { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); assertFalse(connectionManager.nodeConnected(seedNode)); assertFalse(connectionManager.nodeConnected(discoverableNode)); @@ -1131,9 +1138,9 @@ public void testCollectNodes() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(() -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { if (randomBoolean()) { - updateSeedNodes(connection, Arrays.asList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); } CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference> reference = new AtomicReference<>(); @@ -1165,14 +1172,14 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted List discoverableTransports = new CopyOnWriteArrayList<>(); try { final int numDiscoverableNodes = randomIntBetween(5, 20); - List> discoverableNodes = new ArrayList<>(numDiscoverableNodes); + List>> discoverableNodes = new ArrayList<>(numDiscoverableNodes); for (int i = 0; i < numDiscoverableNodes; i++ ) { MockTransportService transportService = startTransport("discoverable_node" + i, knownNodes, Version.CURRENT); - discoverableNodes.add(transportService::getLocalDiscoNode); + discoverableNodes.add(Tuple.tuple("discoverable_node" + i, transportService::getLocalDiscoNode)); discoverableTransports.add(transportService); } - List> seedNodes = randomSubsetOf(discoverableNodes); + List>> seedNodes = randomSubsetOf(discoverableNodes); Collections.shuffle(seedNodes, random()); try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -1221,7 +1228,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted discoverableTransports.add(transportService); connection.addConnectedNode(transportService.getLocalDiscoNode()); } else { - DiscoveryNode node = randomFrom(discoverableNodes).get(); + DiscoveryNode node = randomFrom(discoverableNodes).v2().get(); connection.onNodeDisconnected(node); } } @@ -1269,14 +1276,16 @@ public void testClusterNameIsChecked() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList( () -> seedNode), service, Integer.MAX_VALUE, n -> true, null)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null)) { ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, Collections.singletonList(() -> seedNode)); + updateSeedNodes(connection, seedNodes(seedNode)); assertTrue(connectionManager.nodeConnected(seedNode)); assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); - List> discoveryNodes = - Arrays.asList(otherClusterTransport::getLocalDiscoNode, () -> seedNode); + List>> discoveryNodes = + Arrays.asList( + Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode), + Tuple.tuple(seedNode.toString(), () -> seedNode)); Collections.shuffle(discoveryNodes, random()); updateSeedNodes(connection, discoveryNodes); assertTrue(connectionManager.nodeConnected(seedNode)); @@ -1287,7 +1296,7 @@ public void testClusterNameIsChecked() throws Exception { assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () -> - updateSeedNodes(connection, Arrays.asList(() -> otherClusterTransport.getLocalDiscoNode()))); + updateSeedNodes(connection, Arrays.asList(Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode)))); assertThat(illegalStateException.getMessage(), startsWith("handshake failed, mismatched cluster name [Cluster [otherCluster]]" + " - {other_cluster_discoverable_node}")); @@ -1339,7 +1348,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Collections.singletonList(() -> connectedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { + seedNodes(connectedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { connection.addConnectedNode(connectedNode); for (int i = 0; i < 10; i++) { //always a direct connection as the remote node is already connected @@ -1376,10 +1385,10 @@ public void testLazyResolveTransportAddress() throws Exception { service.start(); service.acceptIncomingRequests(); CountDownLatch multipleResolveLatch = new CountDownLatch(2); - Supplier seedSupplier = () -> { + Tuple> seedSupplier = Tuple.tuple(seedNode.toString(), () -> { multipleResolveLatch.countDown(); return seedNode; - }; + }); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", Arrays.asList(seedSupplier), service, Integer.MAX_VALUE, n -> true, null)) { updateSeedNodes(connection, Arrays.asList(seedSupplier)); @@ -1409,9 +1418,9 @@ public void testProxyMode() throws Exception { threadPool, null, Collections.emptySet())) { service.start(); service.acceptIncomingRequests(); - Supplier seedSupplier = () -> - RemoteClusterAware.buildSeedNode("some-remote-cluster", "node_0:" + randomIntBetween(1, 10000), true); - assertEquals("node_0", seedSupplier.get().getAttributes().get("server_name")); + Tuple> seedSupplier = Tuple.tuple("node_0", () -> + RemoteClusterAware.buildSeedNode("some-remote-cluster", "node_0:" + randomIntBetween(1, 10000), true)); + assertEquals("node_0", seedSupplier.v2().get().getAttributes().get("server_name")); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", Arrays.asList(seedSupplier), service, Integer.MAX_VALUE, n -> true, proxyAddress)) { updateSeedNodes(connection, Arrays.asList(seedSupplier), proxyAddress); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index dfc5d4367b417..34dfc420133c0 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -125,41 +125,42 @@ public void testRemoteClusterSeedSetting() { } public void testBuildRemoteClustersDynamicConfig() throws Exception { - Map>>> map = RemoteClusterService.buildRemoteClustersDynamicConfig( - Settings.builder() - .put("cluster.remote.foo.seeds", "192.168.0.1:8080") - .put("cluster.remote.bar.seeds", "[::1]:9090") - .put("cluster.remote.boom.seeds", "boom-node1.internal:1000") - .put("cluster.remote.boom.proxy", "foo.bar.com:1234") - .put("search.remote.quux.seeds", "quux:9300") - .put("search.remote.quux.proxy", "quux-proxy:19300") - .build()); + Map>>>> map = + RemoteClusterService.buildRemoteClustersDynamicConfig( + Settings.builder() + .put("cluster.remote.foo.seeds", "192.168.0.1:8080") + .put("cluster.remote.bar.seeds", "[::1]:9090") + .put("cluster.remote.boom.seeds", "boom-node1.internal:1000") + .put("cluster.remote.boom.proxy", "foo.bar.com:1234") + .put("search.remote.quux.seeds", "quux:9300") + .put("search.remote.quux.proxy", "quux-proxy:19300") + .build()); assertThat(map.keySet(), containsInAnyOrder(equalTo("foo"), equalTo("bar"), equalTo("boom"), equalTo("quux"))); assertThat(map.get("foo").v2(), hasSize(1)); assertThat(map.get("bar").v2(), hasSize(1)); assertThat(map.get("boom").v2(), hasSize(1)); assertThat(map.get("quux").v2(), hasSize(1)); - DiscoveryNode foo = map.get("foo").v2().get(0).get(); + DiscoveryNode foo = map.get("foo").v2().get(0).v2().get(); assertEquals("", map.get("foo").v1()); assertEquals(foo.getAddress(), new TransportAddress(new InetSocketAddress(InetAddress.getByName("192.168.0.1"), 8080))); assertEquals(foo.getId(), "foo#192.168.0.1:8080"); assertEquals(foo.getVersion(), Version.CURRENT.minimumCompatibilityVersion()); - DiscoveryNode bar = map.get("bar").v2().get(0).get(); + DiscoveryNode bar = map.get("bar").v2().get(0).v2().get(); assertEquals(bar.getAddress(), new TransportAddress(new InetSocketAddress(InetAddress.getByName("[::1]"), 9090))); assertEquals(bar.getId(), "bar#[::1]:9090"); assertEquals("", map.get("bar").v1()); assertEquals(bar.getVersion(), Version.CURRENT.minimumCompatibilityVersion()); - DiscoveryNode boom = map.get("boom").v2().get(0).get(); + DiscoveryNode boom = map.get("boom").v2().get(0).v2().get(); assertEquals(boom.getAddress(), new TransportAddress(TransportAddress.META_ADDRESS, 0)); assertEquals("boom-node1.internal", boom.getHostName()); assertEquals(boom.getId(), "boom#boom-node1.internal:1000"); assertEquals("foo.bar.com:1234", map.get("boom").v1()); assertEquals(boom.getVersion(), Version.CURRENT.minimumCompatibilityVersion()); - DiscoveryNode quux = map.get("quux").v2().get(0).get(); + DiscoveryNode quux = map.get("quux").v2().get(0).v2().get(); assertEquals(quux.getAddress(), new TransportAddress(TransportAddress.META_ADDRESS, 0)); assertEquals("quux", quux.getHostName()); assertEquals(quux.getId(), "quux#quux:9300"); From 46f86b7557b173c2b4da753c94093ce532d3cabb Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 17 Dec 2018 10:50:19 -0800 Subject: [PATCH 40/57] [Painless] Add boxed type to boxed type casts for method/return (#36571) This adds implicit boxed type to boxed types casts for non-def types to create asymmetric casting relative to the def type when calling methods or returning values. This means that a user calling a method taking an Integer can call it with a Byte, Short, etc. legally which matches the way def works. This creates consistency in the casting model that did not previously exist. --- .../painless/AnalyzerCaster.java | 182 ++----- .../elasticsearch/painless/MethodWriter.java | 4 + .../painless/lookup/PainlessCast.java | 9 + .../painless/BoxedCastTests.java | 511 ++++++++++++++++++ 4 files changed, 573 insertions(+), 133 deletions(-) create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java index f00a30a62c4ad..ac21be1f5c079 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java @@ -73,38 +73,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas } else if (expected == Double.class) { return PainlessCast.originalTypetoTargetType(def.class, Double.class, explicit); } - } else if (actual == Object.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Byte.class, true, byte.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Short.class, true, short.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Character.class, true, char.class); - } else if (expected == int.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Integer.class, true, int.class); - } else if (expected == long.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Long.class, true, long.class); - } else if (expected == float.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Float.class, true, float.class); - } else if (expected == double.class && explicit && internal) { - return PainlessCast.unboxTargetType(Object.class, Double.class, true, double.class); - } - } else if (actual == Number.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Byte.class, true, byte.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Short.class, true, short.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Character.class, true, char.class); - } else if (expected == int.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Integer.class, true, int.class); - } else if (expected == long.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Long.class, true, long.class); - } else if (expected == float.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Float.class, true, float.class); - } else if (expected == double.class && explicit && internal) { - return PainlessCast.unboxTargetType(Number.class, Double.class, true, double.class); - } } else if (actual == String.class) { if (expected == char.class && explicit) { return PainlessCast.originalTypetoTargetType(String.class, char.class, true); @@ -140,8 +108,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.boxTargetType(byte.class, byte.class, explicit, byte.class); } else if (expected == Short.class && internal) { return PainlessCast.boxTargetType(byte.class, short.class, explicit, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(byte.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { return PainlessCast.boxTargetType(byte.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { @@ -170,12 +136,8 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(short.class, float.class, explicit); } else if (expected == double.class) { return PainlessCast.originalTypetoTargetType(short.class, double.class, explicit); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(short.class, byte.class, true, byte.class); } else if (expected == Short.class && internal) { return PainlessCast.boxTargetType(short.class, short.class, explicit, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(short.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { return PainlessCast.boxTargetType(short.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { @@ -206,10 +168,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(char.class, float.class, explicit); } else if (expected == double.class) { return PainlessCast.originalTypetoTargetType(char.class, double.class, explicit); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(char.class, byte.class, true, byte.class); - } else if (expected == Short.class && internal) { - return PainlessCast.boxTargetType(char.class, short.class, explicit, short.class); } else if (expected == Character.class && internal) { return PainlessCast.boxTargetType(char.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { @@ -240,12 +198,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(int.class, float.class, explicit); } else if (expected == double.class) { return PainlessCast.originalTypetoTargetType(int.class, double.class, explicit); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(int.class, byte.class, true, byte.class); - } else if (expected == Short.class && explicit && internal) { - return PainlessCast.boxTargetType(int.class, short.class, true, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(int.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { return PainlessCast.boxTargetType(int.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { @@ -274,14 +226,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(long.class, float.class, explicit); } else if (expected == double.class) { return PainlessCast.originalTypetoTargetType(long.class, double.class, explicit); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(long.class, byte.class, true, byte.class); - } else if (expected == Short.class && explicit && internal) { - return PainlessCast.boxTargetType(long.class, short.class, true, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(long.class, char.class, true, char.class); - } else if (expected == Integer.class && explicit && internal) { - return PainlessCast.boxTargetType(long.class, int.class, true, int.class); } else if (expected == Long.class && internal) { return PainlessCast.boxTargetType(long.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { @@ -308,16 +252,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(float.class, long.class, true); } else if (expected == double.class) { return PainlessCast.originalTypetoTargetType(float.class, double.class, explicit); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(float.class, byte.class, true, byte.class); - } else if (expected == Short.class && explicit && internal) { - return PainlessCast.boxTargetType(float.class, short.class, true, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(float.class, char.class, true, char.class); - } else if (expected == Integer.class && explicit && internal) { - return PainlessCast.boxTargetType(float.class, int.class, true, int.class); - } else if (expected == Long.class && explicit && internal) { - return PainlessCast.boxTargetType(float.class, long.class, true, long.class); } else if (expected == Float.class && internal) { return PainlessCast.boxTargetType(float.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { @@ -342,18 +276,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.originalTypetoTargetType(double.class, long.class, true); } else if (expected == float.class && explicit) { return PainlessCast.originalTypetoTargetType(double.class, float.class, true); - } else if (expected == Byte.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, byte.class, true, byte.class); - } else if (expected == Short.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, short.class, true, short.class); - } else if (expected == Character.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, char.class, true, char.class); - } else if (expected == Integer.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, int.class, true, int.class); - } else if (expected == Long.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, long.class, true, long.class); - } else if (expected == Float.class && explicit && internal) { - return PainlessCast.boxTargetType(double.class, float.class, true, float.class); } else if (expected == Double.class && internal) { return PainlessCast.boxTargetType(double.class, double.class, explicit, double.class); } @@ -366,8 +288,6 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.unboxOriginalType(byte.class, byte.class, explicit, byte.class); } else if (expected == short.class && internal) { return PainlessCast.unboxOriginalType(byte.class, short.class, explicit, byte.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(byte.class, char.class, true, byte.class); } else if (expected == int.class && internal) { return PainlessCast.unboxOriginalType(byte.class, int.class, explicit, byte.class); } else if (expected == long.class && internal) { @@ -376,14 +296,20 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.unboxOriginalType(byte.class, float.class, explicit, byte.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(byte.class, double.class, explicit, byte.class); + } else if (expected == Short.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, byte.class, short.class); + } else if (expected == Integer.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, byte.class, int.class); + } else if (expected == Long.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, byte.class, long.class); + } else if (expected == Float.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, byte.class, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, byte.class, double.class); } } else if (actual == Short.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(short.class, byte.class, true, short.class); - } else if (expected == short.class && internal) { + if (expected == short.class && internal) { return PainlessCast.unboxOriginalType(short.class, short.class, explicit, short.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(short.class, char.class, true, short.class); } else if (expected == int.class && internal) { return PainlessCast.unboxOriginalType(short.class, int.class, explicit, short.class); } else if (expected == long.class && internal) { @@ -392,13 +318,17 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.unboxOriginalType(short.class, float.class, explicit, short.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(short.class, double.class, explicit, short.class); + } else if (expected == Integer.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, short.class, int.class); + } else if (expected == Long.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, short.class, long.class); + } else if (expected == Float.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, short.class, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, short.class, double.class); } } else if (actual == Character.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(char.class, byte.class, true, char.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxOriginalType(char.class, short.class, true, char.class); - } else if (expected == char.class && internal) { + if (expected == char.class && internal) { return PainlessCast.unboxOriginalType(char.class, char.class, explicit, char.class); } else if (expected == int.class && internal) { return PainlessCast.unboxOriginalType(char.class, int.class, explicit, char.class); @@ -408,15 +338,17 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.unboxOriginalType(char.class, float.class, explicit, char.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(char.class, double.class, explicit, char.class); + } else if (expected == Integer.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, char.class, int.class); + } else if (expected == Long.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, char.class, long.class); + } else if (expected == Float.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, char.class, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, char.class, double.class); } } else if (actual == Integer.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(int.class, byte.class, true, int.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxOriginalType(int.class, short.class, true, int.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(int.class, char.class, true, int.class); - } else if (expected == int.class && internal) { + if (expected == int.class && internal) { return PainlessCast.unboxOriginalType(int.class, int.class, explicit, int.class); } else if (expected == long.class && internal) { return PainlessCast.unboxOriginalType(int.class, long.class, explicit, int.class); @@ -424,61 +356,45 @@ public static PainlessCast getLegalCast(Location location, Class actual, Clas return PainlessCast.unboxOriginalType(int.class, float.class, explicit, int.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(int.class, double.class, explicit, int.class); + } else if (expected == Long.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, int.class, long.class); + } else if (expected == Float.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, int.class, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, int.class, double.class); } } else if (actual == Long.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(long.class, byte.class, true, long.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxOriginalType(long.class, short.class, true, long.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(long.class, char.class, true, long.class); - } else if (expected == int.class && explicit && internal) { - return PainlessCast.unboxOriginalType(long.class, int.class, true, long.class); - } else if (expected == long.class && internal) { + if (expected == long.class && internal) { return PainlessCast.unboxOriginalType(long.class, long.class, explicit, long.class); } else if (expected == float.class && internal) { return PainlessCast.unboxOriginalType(long.class, float.class, explicit, long.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(long.class, double.class, explicit, long.class); + } else if (expected == Float.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, long.class, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, long.class, double.class); } } else if (actual == Float.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(float.class, byte.class, true, float.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxOriginalType(float.class, short.class, true, float.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(float.class, char.class, true, float.class); - } else if (expected == int.class && explicit && internal) { - return PainlessCast.unboxOriginalType(float.class, int.class, true, float.class); - } else if (expected == long.class && explicit && internal) { - return PainlessCast.unboxOriginalType(float.class, long.class, true, float.class); - } else if (expected == float.class && internal) { + if (expected == float.class && internal) { return PainlessCast.unboxOriginalType(float.class, float.class, explicit, float.class); } else if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(float.class, double.class, explicit, float.class); + } else if (expected == Double.class && internal) { + return PainlessCast.unboxOriginalTypeToBoxTargetType(explicit, float.class, double.class); } } else if (actual == Double.class) { - if (expected == byte.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, byte.class, true, double.class); - } else if (expected == short.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, short.class, true, double.class); - } else if (expected == char.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, char.class, true, double.class); - } else if (expected == int.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, int.class, true, double.class); - } else if (expected == long.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, long.class, true, double.class); - } else if (expected == float.class && explicit && internal) { - return PainlessCast.unboxOriginalType(double.class, float.class, true, double.class); - } else if (expected == double.class && internal) { + if (expected == double.class && internal) { return PainlessCast.unboxOriginalType(double.class, double.class, explicit, double.class); } } - if ( actual == def.class || + if ( + actual == def.class || (actual != void.class && expected == def.class) || - expected.isAssignableFrom(actual) || - (actual.isAssignableFrom(expected) && explicit)) { + expected.isAssignableFrom(actual) || + (actual.isAssignableFrom(expected) && explicit) + ) { return PainlessCast.originalTypetoTargetType(actual, expected, explicit); } else { throw location.createError(new ClassCastException("Cannot cast from " + diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index a2433689db3fe..ea58e7df7b4da 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -154,6 +154,10 @@ public void writeCast(PainlessCast cast) { invokeStatic(UTILITY_TYPE, CHAR_TO_STRING); } else if (cast.originalType == String.class && cast.targetType == char.class) { invokeStatic(UTILITY_TYPE, STRING_TO_CHAR); + } else if (cast.unboxOriginalType != null && cast.boxTargetType != null) { + unbox(getType(cast.unboxOriginalType)); + writeCast(cast.unboxOriginalType, cast.boxTargetType); + box(getType(cast.boxTargetType)); } else if (cast.unboxOriginalType != null) { unbox(getType(cast.unboxOriginalType)); writeCast(cast.originalType, cast.targetType); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java index 98968465d344e..5a3fb848a61d0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java @@ -75,6 +75,15 @@ public static PainlessCast boxTargetType( return new PainlessCast(originalType, targetType, explicitCast, null, null, null, boxTargetType); } + /** Create a cast where the original type is unboxed, cast to a target type, and the target type is boxed. */ + public static PainlessCast unboxOriginalTypeToBoxTargetType(boolean explicitCast, Class unboxOriginalType, Class boxTargetType) { + + Objects.requireNonNull(unboxOriginalType); + Objects.requireNonNull(boxTargetType); + + return new PainlessCast(null, null, explicitCast, unboxOriginalType, null, null, boxTargetType); + } + public final Class originalType; public final Class targetType; public final boolean explicitCast; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java new file mode 100644 index 0000000000000..67a2b683ab6ae --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java @@ -0,0 +1,511 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +public class BoxedCastTests extends ScriptTestCase { + + public void testMethodCallByteToBoxedCasts() { + assertEquals(0, exec("byte u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("byte u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Byte u = Byte.valueOf((byte)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + assertEquals(0, exec("byte u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("byte u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("byte u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Byte u = Byte.valueOf((byte)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Byte u = Byte.valueOf((byte)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + assertEquals(0, exec("def u = (byte)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (byte)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + assertEquals(0, exec("def u = (byte)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (byte)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (byte)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallShortToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("short u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("short u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Short u = Short.valueOf((short)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Short u = Short.valueOf((short)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("short u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("short u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("short u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Short u = Short.valueOf((short)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Short u = Short.valueOf((short)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Short u = Short.valueOf((short)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (short)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (short)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (short)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (short)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (short)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallCharacterToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("char u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("char u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Character u = Character.valueOf((char)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Character u = Character.valueOf((char)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("char u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("char u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("char u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Character u = Character.valueOf((char)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Character u = Character.valueOf((char)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Character u = Character.valueOf((char)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (char)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (char)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (char)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (char)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (char)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallIntegerToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("int u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("int u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Integer u = Integer.valueOf((int)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Integer u = Integer.valueOf((int)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (int)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (int)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallLongToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("long u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("long u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Long u = Long.valueOf((long)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Long u = Long.valueOf((long)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (long)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (long)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallFloatToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("float u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("float u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Float u = Float.valueOf((float)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Float u = Float.valueOf((float)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("float u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("float u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("float u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Float u = Float.valueOf((float)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("Float u = Float.valueOf((float)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Float u = Float.valueOf((float)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (float)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (float)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (float)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (float)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (float)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } + + public void testMethodCallDoubleToBoxedCasts() { + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("double u = 1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Long b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Double u = Double.valueOf((double)1); Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("double u = 1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("double u = 1; def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("Double u = Double.valueOf((double)1); def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("Double u = Double.valueOf((double)1); def b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Byte b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Short b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Character b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Integer b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Long b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; Float b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (double)1; Double b = Double.valueOf((double)1); b.compareTo(u);")); + + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Byte.valueOf((byte)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Short.valueOf((short)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Character.valueOf((char)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Integer.valueOf((int)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Long.valueOf((long)1); b.compareTo(u);")); + expectScriptThrows(ClassCastException.class, + () -> exec("def u = (double)1; def b = Float.valueOf((float)1); b.compareTo(u);")); + assertEquals(0, exec("def u = (double)1; def b = Double.valueOf((double)1); b.compareTo(u);")); + } +} From c522341bf3736a30325115c7eab1bcebedaef01c Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 17 Dec 2018 21:02:59 +0100 Subject: [PATCH 41/57] SNAPSHOTS: Adjust BwC Versions in Restore Logic (#36718) * Re-enables bwc tests with adjusted version conditions now that #36397 enables concurrent snapshots in 6.6+ --- build.gradle | 4 ++-- .../java/org/elasticsearch/cluster/RestoreInProgress.java | 6 +++--- .../org/elasticsearch/cluster/routing/RecoverySource.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 36412c047a761..7e067b8997805 100644 --- a/build.gradle +++ b/build.gradle @@ -163,8 +163,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/36659" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index c229a826ee8ee..d71a3f94d4063 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -46,7 +46,7 @@ public class RestoreInProgress extends AbstractNamedDiffable implements Custom, Iterable { /** - * Fallback UUID used for restore operations that were started before v7.0 and don't have a uuid in the cluster state. + * Fallback UUID used for restore operations that were started before v6.6 and don't have a uuid in the cluster state. */ public static final String BWC_UUID = new UUID(0, 0).toString(); @@ -436,7 +436,7 @@ public RestoreInProgress(StreamInput in) throws IOException { final ImmutableOpenMap.Builder entriesBuilder = ImmutableOpenMap.builder(count); for (int i = 0; i < count; i++) { final String uuid; - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { uuid = in.readString(); } else { uuid = BWC_UUID; @@ -468,7 +468,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(entries.size()); for (ObjectCursor v : entries.values()) { Entry entry = v.value; - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { out.writeString(entry.uuid); } entry.snapshot().writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java index 3654d66ad58cd..25a605088ef66 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java @@ -222,7 +222,7 @@ public SnapshotRecoverySource(String restoreUUID, Snapshot snapshot, Version ver } SnapshotRecoverySource(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { restoreUUID = in.readString(); } else { restoreUUID = RestoreInProgress.BWC_UUID; @@ -250,7 +250,7 @@ public Version version() { @Override protected void writeAdditionalFields(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { out.writeString(restoreUUID); } snapshot.writeTo(out); From 6b4c8c85b2df4c9a519dfaba6146c31c0dcc37ad Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 17 Dec 2018 14:10:13 -0600 Subject: [PATCH 42/57] ingest: fix on_failure with Drop processor (#36686) This commit allows a document to be dropped when a Drop processor is used in the on_failure fork of the processor chain. Fixes #36151 --- .../test/ingest/220_drop_processor.yml | 41 +++++++++++++++++++ .../ingest/CompoundProcessor.java | 15 +++++-- .../ingest/CompoundProcessorTests.java | 29 +++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml index 3be038aca2499..accc30faa21e7 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml @@ -57,3 +57,44 @@ teardown: type: test id: 2 - match: { _source.foo: "blub" } + +--- +"Test Drop Processor On Failure": +- do: + ingest.put_pipeline: + id: "my_pipeline_with_failure" + body: > + { + "description" : "pipeline with on failure drop", + "processors": [ + { + "fail": { + "message": "failed", + "on_failure": [ + { + "drop": {} + } + ] + } + } + ] + } +- match: { acknowledged: true } + +- do: + index: + index: test + type: test + id: 3 + pipeline: "my_pipeline_with_failure" + body: { + foo: "bar" + } + +- do: + catch: missing + get: + index: test + type: test + id: 3 +- match: { found: false } diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 3b8281bd471d2..a095d7647d90f 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -134,7 +134,9 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { if (onFailureProcessors.isEmpty()) { throw compoundProcessorException; } else { - executeOnFailure(ingestDocument, compoundProcessorException); + if (executeOnFailure(ingestDocument, compoundProcessorException) == false) { + return null; + } break; } } finally { @@ -145,13 +147,17 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { return ingestDocument; } - - void executeOnFailure(IngestDocument ingestDocument, ElasticsearchException exception) throws Exception { + /** + * @return true if execution should continue, false if document is dropped. + */ + boolean executeOnFailure(IngestDocument ingestDocument, ElasticsearchException exception) throws Exception { try { putFailureMetadata(ingestDocument, exception); for (Processor processor : onFailureProcessors) { try { - processor.execute(ingestDocument); + if (processor.execute(ingestDocument) == null) { + return false; + } } catch (Exception e) { throw newCompoundProcessorException(e, processor.getType(), processor.getTag()); } @@ -159,6 +165,7 @@ void executeOnFailure(IngestDocument ingestDocument, ElasticsearchException exce } finally { removeFailureMetadata(ingestDocument); } + return true; } private void putFailureMetadata(IngestDocument ingestDocument, ElasticsearchException cause) { diff --git a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java index dabcae533a0bf..24e3dcd76774b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java @@ -129,6 +129,35 @@ public void testSingleProcessorWithOnFailureProcessor() throws Exception { assertThat(processor2.getInvokedCounter(), equalTo(1)); } + public void testSingleProcessorWithOnFailureDropProcessor() throws Exception { + TestProcessor processor1 = new TestProcessor("id", "first", ingestDocument -> {throw new RuntimeException("error");}); + Processor processor2 = new Processor() { + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + //Simulates the drop processor + return null; + } + + @Override + public String getType() { + return "drop"; + } + + @Override + public String getTag() { + return null; + } + }; + + LongSupplier relativeTimeProvider = mock(LongSupplier.class); + when(relativeTimeProvider.getAsLong()).thenReturn(0L); + CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor1), + Collections.singletonList(processor2), relativeTimeProvider); + assertNull(compoundProcessor.execute(ingestDocument)); + assertThat(processor1.getInvokedCounter(), equalTo(1)); + assertStats(compoundProcessor, 1, 1, 0); + } + public void testSingleProcessorWithNestedFailures() throws Exception { TestProcessor processor = new TestProcessor("id", "first", ingestDocument -> {throw new RuntimeException("error");}); TestProcessor processorToFail = new TestProcessor("id2", "second", ingestDocument -> { From 992b531ea9baba5bb305ffc1596f2c80a2390d5d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 17 Dec 2018 13:19:32 -0700 Subject: [PATCH 43/57] Initialize startup `CcrRepositories` (#36730) Currently, the CcrRepositoryManger only listens for settings updates and installs new repositories. It does not install the repositories that are in the initial settings. This commit, modifies the manager to install the initial repositories. Additionally, it modifies the ccr integration test to configure the remote leader node at startup, instead of using a settings update. --- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 5 +- .../xpack/ccr/CcrRepositoryManager.java | 68 +++++++++++++++---- .../elasticsearch/xpack/CcrIntegTestCase.java | 24 +++---- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 70d4905d94375..58ba11e4d0488 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ccr; -import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; @@ -111,7 +110,6 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final boolean enabled; private final Settings settings; private final CcrLicenseChecker ccrLicenseChecker; - private final SetOnce repositoryManager = new SetOnce<>(); private Client client; /** @@ -152,10 +150,9 @@ public Collection createComponents( return emptyList(); } - this.repositoryManager.set(new CcrRepositoryManager(settings, clusterService, client)); - return Arrays.asList( ccrLicenseChecker, + new CcrRepositoryManager(settings, clusterService, client), new AutoFollowCoordinator(client, clusterService, ccrLicenseChecker, threadPool::relativeTimeInMillis) ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRepositoryManager.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRepositoryManager.java index a1504ff2f8acd..54403df367809 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRepositoryManager.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRepositoryManager.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.ccr.action.repositories.DeleteInternalCcrRepositoryAction; @@ -18,31 +19,70 @@ import org.elasticsearch.xpack.ccr.action.repositories.PutInternalCcrRepositoryRequest; import org.elasticsearch.xpack.ccr.repository.CcrRepository; +import java.io.IOException; import java.util.List; +import java.util.Set; -class CcrRepositoryManager extends RemoteClusterAware { +class CcrRepositoryManager extends AbstractLifecycleComponent { private final Client client; + private final RemoteSettingsUpdateListener updateListener; CcrRepositoryManager(Settings settings, ClusterService clusterService, Client client) { super(settings); this.client = client; - listenForUpdates(clusterService.getClusterSettings()); + updateListener = new RemoteSettingsUpdateListener(settings); + updateListener.listenForUpdates(clusterService.getClusterSettings()); } @Override - protected void updateRemoteCluster(String clusterAlias, List addresses, String proxyAddress) { - String repositoryName = CcrRepository.NAME_PREFIX + clusterAlias; - if (addresses.isEmpty()) { - DeleteInternalCcrRepositoryRequest request = new DeleteInternalCcrRepositoryRequest(repositoryName); - PlainActionFuture f = PlainActionFuture.newFuture(); - client.execute(DeleteInternalCcrRepositoryAction.INSTANCE, request, f); - assert f.isDone() : "Should be completed as it is executed synchronously"; - } else { - ActionRequest request = new PutInternalCcrRepositoryRequest(repositoryName, CcrRepository.TYPE); - PlainActionFuture f = PlainActionFuture.newFuture(); - client.execute(PutInternalCcrRepositoryAction.INSTANCE, request, f); - assert f.isDone() : "Should be completed as it is executed synchronously"; + protected void doStart() { + updateListener.init(); + } + + @Override + protected void doStop() { + } + + @Override + protected void doClose() throws IOException { + } + + private void putRepository(String repositoryName) { + ActionRequest request = new PutInternalCcrRepositoryRequest(repositoryName, CcrRepository.TYPE); + PlainActionFuture f = PlainActionFuture.newFuture(); + client.execute(PutInternalCcrRepositoryAction.INSTANCE, request, f); + assert f.isDone() : "Should be completed as it is executed synchronously"; + } + + private void deleteRepository(String repositoryName) { + DeleteInternalCcrRepositoryRequest request = new DeleteInternalCcrRepositoryRequest(repositoryName); + PlainActionFuture f = PlainActionFuture.newFuture(); + client.execute(DeleteInternalCcrRepositoryAction.INSTANCE, request, f); + assert f.isDone() : "Should be completed as it is executed synchronously"; + } + + private class RemoteSettingsUpdateListener extends RemoteClusterAware { + + private RemoteSettingsUpdateListener(Settings settings) { + super(settings); + } + + void init() { + Set clusterAliases = buildRemoteClustersDynamicConfig(settings).keySet(); + for (String clusterAlias : clusterAliases) { + putRepository(CcrRepository.NAME_PREFIX + clusterAlias); + } + } + + @Override + protected void updateRemoteCluster(String clusterAlias, List addresses, String proxy) { + String repositoryName = CcrRepository.NAME_PREFIX + clusterAlias; + if (addresses.isEmpty()) { + deleteRepository(repositoryName); + } else { + putRepository(repositoryName); + } } } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 5abe852ca5ff0..8865c53691786 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -117,27 +116,23 @@ public final void startClusters() throws Exception { } stopClusters(); - NodeConfigurationSource nodeConfigurationSource = createNodeConfigurationSource(); Collection> mockPlugins = Arrays.asList(ESIntegTestCase.TestSeedPlugin.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class, getTestTransportPlugin()); InternalTestCluster leaderCluster = new InternalTestCluster(randomLong(), createTempDir(), true, true, numberOfNodesPerCluster(), - numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, "leader", mockPlugins, + numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), createNodeConfigurationSource(null), 0, "leader", mockPlugins, Function.identity()); + leaderCluster.beforeTest(random(), 0.0D); + leaderCluster.ensureAtLeastNumDataNodes(numberOfNodesPerCluster()); + + String address = leaderCluster.getDataNodeInstance(TransportService.class).boundAddress().publishAddress().toString(); InternalTestCluster followerCluster = new InternalTestCluster(randomLong(), createTempDir(), true, true, numberOfNodesPerCluster(), - numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, "follower", mockPlugins, - Function.identity()); + numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), createNodeConfigurationSource(address), 0, "follower", + mockPlugins, Function.identity()); clusterGroup = new ClusterGroup(leaderCluster, followerCluster); - leaderCluster.beforeTest(random(), 0.0D); - leaderCluster.ensureAtLeastNumDataNodes(numberOfNodesPerCluster()); followerCluster.beforeTest(random(), 0.0D); followerCluster.ensureAtLeastNumDataNodes(numberOfNodesPerCluster()); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - String address = leaderCluster.getDataNodeInstance(TransportService.class).boundAddress().publishAddress().toString(); - updateSettingsRequest.persistentSettings(Settings.builder().put("cluster.remote.leader_cluster.seeds", address)); - assertAcked(followerClient().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); } /** @@ -175,7 +170,7 @@ public void afterTest() throws Exception { } } - private NodeConfigurationSource createNodeConfigurationSource() { + private NodeConfigurationSource createNodeConfigurationSource(String leaderSeedAddress) { Settings.Builder builder = Settings.builder(); builder.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE); // Default the watermarks to absurdly low to prevent the tests @@ -195,6 +190,9 @@ private NodeConfigurationSource createNodeConfigurationSource() { builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false); builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); + if (leaderSeedAddress != null) { + builder.put("cluster.remote.leader_cluster.seeds", leaderSeedAddress); + } return new NodeConfigurationSource() { @Override public Settings nodeSettings(int nodeOrdinal) { From e2489a7866d4da8c7a47ce0dabc66aba91bffddd Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 17 Dec 2018 21:19:39 +0100 Subject: [PATCH 44/57] [TEST] fix float comparison in RandomObjects#getExpectedParsedValue This commit fixes a test bug introduced with #36597. This caused some test failure as stored field values comparisons would not work when CBOR xcontent type was used. Closes #29080 --- .../main/java/org/elasticsearch/test/RandomObjects.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java index c81d0810f08be..4669284685c11 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java +++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java @@ -135,13 +135,16 @@ public static Object getExpectedParsedValue(XContentType xContentType, Object va } } if (value instanceof Float) { + if (xContentType == XContentType.CBOR) { + //with CBOR we get back a float + return value; + } if (xContentType == XContentType.SMILE) { //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float) return ((Float)value).doubleValue(); - } else { - //with JSON AND YAML we get back a double, but with float precision. - return Double.parseDouble(value.toString()); } + //with JSON AND YAML we get back a double, but with float precision. + return Double.parseDouble(value.toString()); } if (value instanceof Byte) { return ((Byte)value).intValue(); From 69030431d31de820c917516dfe2094854745b8a2 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 17 Dec 2018 14:38:14 -0600 Subject: [PATCH 45/57] [Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320) This commit exposes lucene's LatLonShape field as the default type in GeoShapeFieldMapper. To use the new indexing approach, simply set "type" : "geo_shape" in the mappings without setting any of the strategy, precision, tree_levels, or distance_error_pct parameters. Note the following when using the new indexing approach: * geo_shape query does not support querying by MULTIPOINT. * LINESTRING and MULTILINESTRING queries do not yet support WITHIN relation. * CONTAINS relation is not yet supported. The tree, precision, tree_levels, distance_error_pct, and points_only parameters are deprecated. --- .../mapping/types/geo-shape.asciidoc | 186 +++-- .../migration/migrate_7_0/mappings.asciidoc | 16 + .../query-dsl/geo-shape-query.asciidoc | 5 +- .../common/geo/ShapeRelation.java | 12 + .../builders/GeometryCollectionBuilder.java | 3 - .../common/geo/parsers/GeoJsonParser.java | 24 +- .../common/geo/parsers/GeoWKTParser.java | 13 +- .../common/geo/parsers/ShapeParser.java | 4 +- .../index/mapper/BaseGeoShapeFieldMapper.java | 336 +++++++++ .../index/mapper/GeoShapeFieldMapper.java | 610 ++------------- .../mapper/LegacyGeoShapeFieldMapper.java | 596 +++++++++++++++ .../index/query/GeoShapeQueryBuilder.java | 117 ++- .../elasticsearch/indices/IndicesModule.java | 8 +- .../common/geo/GeoJsonShapeParserTests.java | 8 +- .../common/geo/GeoWKTShapeParserTests.java | 19 +- .../index/mapper/ExternalMapper.java | 21 +- .../ExternalValuesMapperIntegrationIT.java | 6 +- .../mapper/GeoShapeFieldMapperTests.java | 452 ++--------- .../index/mapper/GeoShapeFieldTypeTests.java | 52 +- .../LegacyGeoShapeFieldMapperTests.java | 714 ++++++++++++++++++ .../mapper/LegacyGeoShapeFieldTypeTests.java | 86 +++ .../query/GeoShapeQueryBuilderTests.java | 75 +- .../query/LegacyGeoShapeFieldQueryTests.java | 94 +++ .../index/query/MatchQueryBuilderTests.java | 1 + .../query/QueryStringQueryBuilderTests.java | 6 + .../elasticsearch/search/geo/GeoFilterIT.java | 1 + .../search/geo/GeoShapeIntegrationIT.java | 25 +- .../search/geo/GeoShapeQueryTests.java | 186 ++++- .../geo/LegacyGeoShapeIntegrationIT.java | 170 +++++ .../test/geo/RandomShapeGenerator.java | 2 + .../test/AbstractBuilderTestCase.java | 20 +- 31 files changed, 2635 insertions(+), 1233 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 2f51465d1109f..8efb184afa6ba 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -21,48 +21,59 @@ type. |======================================================================= |Option |Description| Default -|`tree` |Name of the PrefixTree implementation to be used: `geohash` for -GeohashPrefixTree and `quadtree` for QuadPrefixTree. -| `geohash` - -|`precision` |This parameter may be used instead of `tree_levels` to set -an appropriate value for the `tree_levels` parameter. The value -specifies the desired precision and Elasticsearch will calculate the -best tree_levels value to honor this precision. The value should be a -number followed by an optional distance unit. Valid distance units -include: `in`, `inch`, `yd`, `yard`, `mi`, `miles`, `km`, `kilometers`, -`m`,`meters`, `cm`,`centimeters`, `mm`, `millimeters`. +|`tree |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree +implementation to be used: `geohash` for GeohashPrefixTree and `quadtree` +for QuadPrefixTree. Note: This parameter is only relevant for `term` and +`recursive` strategies. +| `quadtree` + +|`precision` |deprecated[6.6, PrefixTrees no longer used] This parameter may +be used instead of `tree_levels` to set an appropriate value for the +`tree_levels` parameter. The value specifies the desired precision and +Elasticsearch will calculate the best tree_levels value to honor this +precision. The value should be a number followed by an optional distance +unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, +`miles`, `km`, `kilometers`, `m`,`meters`, `cm`,`centimeters`, `mm`, +`millimeters`. Note: This parameter is only relevant for `term` and +`recursive` strategies. | `50m` -|`tree_levels` |Maximum number of layers to be used by the PrefixTree. -This can be used to control the precision of shape representations and -therefore how many terms are indexed. Defaults to the default value of -the chosen PrefixTree implementation. Since this parameter requires a -certain level of understanding of the underlying implementation, users -may use the `precision` parameter instead. However, Elasticsearch only -uses the tree_levels parameter internally and this is what is returned -via the mapping API even if you use the precision parameter. +|`tree_levels` |deprecated[6.6, PrefixTrees no longer used] Maximum number +of layers to be used by the PrefixTree. This can be used to control the +precision of shape representations andtherefore how many terms are +indexed. Defaults to the default value of the chosen PrefixTree +implementation. Since this parameter requires a certain level of +understanding of the underlying implementation, users may use the +`precision` parameter instead. However, Elasticsearch only uses the +tree_levels parameter internally and this is what is returned via the +mapping API even if you use the precision parameter. Note: This parameter +is only relevant for `term` and `recursive` strategies. | various -|`strategy` |The strategy parameter defines the approach for how to -represent shapes at indexing and search time. It also influences the -capabilities available so it is recommended to let Elasticsearch set -this parameter automatically. There are two strategies available: -`recursive` and `term`. Term strategy supports point types only (the -`points_only` parameter will be automatically set to true) while -Recursive strategy supports all shape types. (IMPORTANT: see -<> for more detailed information) +|`strategy` |deprecated[6.6, PrefixTrees no longer used] The strategy +parameter defines the approach for how to represent shapes at indexing +and search time. It also influences the capabilities available so it +is recommended to let Elasticsearch set this parameter automatically. +There are two strategies available: `recursive`, and `term`. +Recursive and Term strategies are deprecated and will be removed in a +future version. While they are still available, the Term strategy +supports point types only (the `points_only` parameter will be +automatically set to true) while Recursive strategy supports all +shape types. (IMPORTANT: see <> for more +detailed information about these strategies) | `recursive` -|`distance_error_pct` |Used as a hint to the PrefixTree about how -precise it should be. Defaults to 0.025 (2.5%) with 0.5 as the maximum -supported value. PERFORMANCE NOTE: This value will default to 0 if a `precision` or -`tree_level` definition is explicitly defined. This guarantees spatial precision -at the level defined in the mapping. This can lead to significant memory usage -for high resolution shapes with low error (e.g., large shapes at 1m with < 0.001 error). -To improve indexing performance (at the cost of query accuracy) explicitly define -`tree_level` or `precision` along with a reasonable `distance_error_pct`, noting -that large shapes will have greater false positives. +|`distance_error_pct` |deprecated[6.6, PrefixTrees no longer used] Used as a +hint to the PrefixTree about how precise it should be. Defaults to 0.025 (2.5%) +with 0.5 as the maximum supported value. PERFORMANCE NOTE: This value will +default to 0 if a `precision` or `tree_level` definition is explicitly defined. +This guarantees spatial precision at the level defined in the mapping. This can +lead to significant memory usage for high resolution shapes with low error +(e.g., large shapes at 1m with < 0.001 error). To improve indexing performance +(at the cost of query accuracy) explicitly define `tree_level` or `precision` +along with a reasonable `distance_error_pct`, noting that large shapes will have +greater false positives. Note: This parameter is only relevant for `term` and +`recursive` strategies. | `0.025` |`orientation` |Optionally define how to interpret vertex order for @@ -77,13 +88,13 @@ sets vertex order for the coordinate list of a geo_shape field but can be overridden in each individual GeoJSON or WKT document. | `ccw` -|`points_only` |Setting this option to `true` (defaults to `false`) configures -the `geo_shape` field type for point shapes only (NOTE: Multi-Points are not -yet supported). This optimizes index and search performance for the `geohash` and -`quadtree` when it is known that only points will be indexed. At present geo_shape -queries can not be executed on `geo_point` field types. This option bridges the gap -by improving point performance on a `geo_shape` field so that `geo_shape` queries are -optimal on a point only field. +|`points_only` |deprecated[6.6, PrefixTrees no longer used] Setting this option to +`true` (defaults to `false`) configures the `geo_shape` field type for point +shapes only (NOTE: Multi-Points are not yet supported). This optimizes index and +search performance for the `geohash` and `quadtree` when it is known that only points +will be indexed. At present geo_shape queries can not be executed on `geo_point` +field types. This option bridges the gap by improving point performance on a +`geo_shape` field so that `geo_shape` queries are optimal on a point only field. | `false` |`ignore_malformed` |If true, malformed GeoJSON or WKT shapes are ignored. If @@ -100,16 +111,35 @@ and reject the whole document. |======================================================================= + +[[geoshape-indexing-approach]] +[float] +==== Indexing approach +GeoShape types are indexed by decomposing the shape into a triangular mesh and +indexing each triangle as a 7 dimension point in a BKD tree. This provides +near perfect spatial resolution (down to 1e-7 decimal degree precision) since all +spatial relations are computed using an encoded vector representation of the +original shape instead of a raster-grid representation as used by the +<> indexing approach. Performance of the tessellator primarily +depends on the number of vertices that define the polygon/multi-polyogn. While +this is the default indexing technique prefix trees can still be used by setting +the `tree` or `strategy` parameters according to the appropriate +<>. Note that these parameters are now deprecated +and will be removed in a future version. + [[prefix-trees]] [float] ==== Prefix trees -To efficiently represent shapes in the index, Shapes are converted into -a series of hashes representing grid squares (commonly referred to as "rasters") -using implementations of a PrefixTree. The tree notion comes from the fact that -the PrefixTree uses multiple grid layers, each with an increasing level of -precision to represent the Earth. This can be thought of as increasing the level -of detail of a map or image at higher zoom levels. +deprecated[6.6, PrefixTrees no longer used] To efficiently represent shapes in +an inverted index, Shapes are converted into a series of hashes representing +grid squares (commonly referred to as "rasters") using implementations of a +PrefixTree. The tree notion comes from the fact that the PrefixTree uses multiple +grid layers, each with an increasing level of precision to represent the Earth. +This can be thought of as increasing the level of detail of a map or image at higher +zoom levels. Since this approach causes precision issues with indexed shape, it has +been deprecated in favor of a vector indexing approach that indexes the shapes as a +triangular mesh (see <>). Multiple PrefixTree implementations are provided: @@ -131,9 +161,10 @@ number of levels for the quad trees in Elasticsearch is 29; the default is 21. [[spatial-strategy]] [float] ===== Spatial strategies -The PrefixTree implementations rely on a SpatialStrategy for decomposing -the provided Shape(s) into approximated grid squares. Each strategy answers -the following: +deprecated[6.6, PrefixTrees no longer used] The indexing implementation +selected relies on a SpatialStrategy for choosing how to decompose the shapes +(either as grid squares or a tessellated triangular mesh). Each strategy +answers the following: * What type of Shapes can be indexed? * What types of Query Operations and Shapes can be used? @@ -146,7 +177,7 @@ are provided: |======================================================================= |Strategy |Supported Shapes |Supported Queries |Multiple Shapes -|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes +|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes |`term` |<> |`INTERSECTS` |Yes |======================================================================= @@ -154,13 +185,13 @@ are provided: [float] ===== Accuracy -Geo_shape does not provide 100% accuracy and depending on how it is configured -it may return some false positives for `INTERSECTS`, `WITHIN` and `CONTAINS` -queries, and some false negatives for `DISJOINT` queries. To mitigate this, it -is important to select an appropriate value for the tree_levels parameter and -to adjust expectations accordingly. For example, a point may be near the border -of a particular grid cell and may thus not match a query that only matches the -cell right next to it -- even though the shape is very close to the point. +`Recursive` and `Term` strategies do not provide 100% accuracy and depending on +how they are configured it may return some false positives for `INTERSECTS`, +`WITHIN` and `CONTAINS` queries, and some false negatives for `DISJOINT` queries. +To mitigate this, it is important to select an appropriate value for the tree_levels +parameter and to adjust expectations accordingly. For example, a point may be near +the border of a particular grid cell and may thus not match a query that only matches +the cell right next to it -- even though the shape is very close to the point. [float] ===== Example @@ -173,9 +204,7 @@ PUT /example "doc": { "properties": { "location": { - "type": "geo_shape", - "tree": "quadtree", - "precision": "100m" + "type": "geo_shape" } } } @@ -185,22 +214,23 @@ PUT /example // CONSOLE // TESTSETUP -This mapping maps the location field to the geo_shape type using the -quad_tree implementation and a precision of 100m. Elasticsearch translates -this into a tree_levels setting of 20. +This mapping definition maps the location field to the geo_shape +type using the default vector implementation. It provides +approximately 1e-7 decimal degree precision. [float] -===== Performance considerations +===== Performance considerations with Prefix Trees -Elasticsearch uses the paths in the prefix tree as terms in the index -and in queries. The higher the level is (and thus the precision), the -more terms are generated. Of course, calculating the terms, keeping them in +deprecated[6.6, PrefixTrees no longer used] With prefix trees, +Elasticsearch uses the paths in the tree as terms in the inverted index +and in queries. The higher the level (and thus the precision), the more +terms are generated. Of course, calculating the terms, keeping them in memory, and storing them on disk all have a price. Especially with higher -tree levels, indices can become extremely large even with a modest -amount of data. Additionally, the size of the features also matters. -Big, complex polygons can take up a lot of space at higher tree levels. -Which setting is right depends on the use case. Generally one trades off -accuracy against index size and query performance. +tree levels, indices can become extremely large even with a modest amount +of data. Additionally, the size of the features also matters. Big, complex +polygons can take up a lot of space at higher tree levels. Which setting +is right depends on the use case. Generally one trades off accuracy against +index size and query performance. The defaults in Elasticsearch for both implementations are a compromise between index size and a reasonable level of precision of 50m at the @@ -598,7 +628,10 @@ POST /example/doc ===== Circle Elasticsearch supports a `circle` type, which consists of a center -point with a radius: +point with a radius. Note that this circle representation can only +be indexed when using the `recursive` Prefix Tree strategy. For +the default <> circles should be approximated using +a `POLYGON`. [source,js] -------------------------------------------------- @@ -612,6 +645,7 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[skip:not supported in default] Note: The inner `radius` field is required. If not specified, then the units of the `radius` will default to `METERS`. diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 5ee1615796c98..f08ea3ab89c1d 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -52,3 +52,19 @@ as a better alternative. An error will now be thrown when unknown configuration options are provided to similarities. Such unknown parameters were ignored before. + +[float] +==== deprecated `geo_shape` Prefix Tree indexing + +`geo_shape` types now default to using a vector indexing approach based on Lucene's new +`LatLonShape` field type. This indexes shapes as a triangular mesh instead of decomposing +them into individual grid cells. To index using legacy prefix trees `recursive` or `term` +strategy must be explicitly defined. Note that these strategies are now deprecated and will +be removed in a future version. + +[float] +==== deprecated `geo_shape` parameters + +The following type parameters are deprecated for the `geo_shape` field type: `tree`, +`precision`, `tree_levels`, `distance_error_pct`, `points_only`, and `strategy`. They +will be removed in a future version. \ No newline at end of file diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc index 4e00a2f49b475..f796881d520c6 100644 --- a/docs/reference/query-dsl/geo-shape-query.asciidoc +++ b/docs/reference/query-dsl/geo-shape-query.asciidoc @@ -7,7 +7,7 @@ Requires the <>. The `geo_shape` query uses the same grid square representation as the `geo_shape` mapping to find documents that have a shape that intersects -with the query shape. It will also use the same PrefixTree configuration +with the query shape. It will also use the same Prefix Tree configuration as defined for the field mapping. The query supports two ways of defining the query shape, either by @@ -157,7 +157,8 @@ has nothing in common with the query geometry. * `WITHIN` - Return all documents whose `geo_shape` field is within the query geometry. * `CONTAINS` - Return all documents whose `geo_shape` field -contains the query geometry. +contains the query geometry. Note: this is only supported using the +`recursive` Prefix Tree Strategy deprecated[6.6] [float] ==== Ignore Unmapped diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java index e83e18ce43255..e2e177c8f0fd2 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; +import org.apache.lucene.document.LatLonShape.QueryRelation; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -62,6 +63,17 @@ public static ShapeRelation getRelationByName(String name) { return null; } + /** Maps ShapeRelation to Lucene's LatLonShapeRelation */ + public QueryRelation getLuceneRelation() { + switch (this) { + case INTERSECTS: return QueryRelation.INTERSECTS; + case DISJOINT: return QueryRelation.DISJOINT; + case WITHIN: return QueryRelation.WITHIN; + default: + throw new IllegalArgumentException("ShapeRelation [" + this + "] not supported"); + } + } + public String getRelationName() { return relationName; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index b6e94c012c603..fdf7073bd7454 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -197,9 +197,6 @@ public Object buildLucene() { } } - if (shapes.size() == 1) { - return shapes.get(0); - } return shapes.toArray(new Object[shapes.size()]); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java index 4f0586711e439..b008786ed9211 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java @@ -25,10 +25,11 @@ import org.elasticsearch.common.geo.builders.CircleBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentSubParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -41,17 +42,22 @@ * complies with geojson specification: https://tools.ietf.org/html/rfc7946 */ abstract class GeoJsonParser { - protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) + protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) throws IOException { GeoShapeType shapeType = null; DistanceUnit.Distance radius = null; CoordinateNode coordinateNode = null; GeometryCollectionBuilder geometryCollections = null; - ShapeBuilder.Orientation requestedOrientation = - (shapeMapper == null) ? ShapeBuilder.Orientation.RIGHT : shapeMapper.fieldType().orientation(); - Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); - Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); + Orientation orientation = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() + : shapeMapper.orientation(); + Explicit coerce = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.COERCE + : shapeMapper.coerce(); + Explicit ignoreZValue = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE + : shapeMapper.ignoreZValue(); String malformedException = null; @@ -102,7 +108,7 @@ protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper s malformedException = "cannot have [" + ShapeParser.FIELD_ORIENTATION + "] with type set to [" + shapeType + "]"; } subParser.nextToken(); - requestedOrientation = ShapeBuilder.Orientation.fromString(subParser.text()); + orientation = ShapeBuilder.Orientation.fromString(subParser.text()); } else { subParser.nextToken(); subParser.skipChildren(); @@ -128,7 +134,7 @@ protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper s return geometryCollections; } - return shapeType.getBuilder(coordinateNode, radius, requestedOrientation, coerce.value()); + return shapeType.getBuilder(coordinateNode, radius, orientation, coerce.value()); } /** @@ -202,7 +208,7 @@ private static Coordinate parseCoordinate(XContentParser parser, boolean ignoreZ * @return Geometry[] geometries of the GeometryCollection * @throws IOException Thrown if an error occurs while reading from the XContentParser */ - static GeometryCollectionBuilder parseGeometries(XContentParser parser, GeoShapeFieldMapper mapper) throws + static GeometryCollectionBuilder parseGeometries(XContentParser parser, BaseGeoShapeFieldMapper mapper) throws IOException { if (parser.currentToken() != XContentParser.Token.START_ARRAY) { throw new ElasticsearchParseException("geometries must be an array of geojson objects"); diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java index e1d990f0cff25..bf26980c92651 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java @@ -34,7 +34,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -63,7 +63,7 @@ public class GeoWKTParser { // no instance private GeoWKTParser() {} - public static ShapeBuilder parse(XContentParser parser, final GeoShapeFieldMapper shapeMapper) + public static ShapeBuilder parse(XContentParser parser, final BaseGeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { return parseExpectedType(parser, null, shapeMapper); } @@ -75,12 +75,12 @@ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoSha /** throws an exception if the parsed geometry type does not match the expected shape type */ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoShapeType shapeType, - final GeoShapeFieldMapper shapeMapper) + final BaseGeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { try (StringReader reader = new StringReader(parser.text())) { - Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : + Explicit ignoreZValue = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); - Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); + Explicit coerce = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); // setup the tokenizer; configured to read words w/o numbers StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.resetSyntax(); @@ -257,7 +257,8 @@ private static PolygonBuilder parsePolygon(StreamTokenizer stream, final boolean if (nextEmptyOrOpen(stream).equals(EMPTY)) { return null; } - PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), ShapeBuilder.Orientation.RIGHT); + PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), + BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value()); while (nextCloserOrComma(stream).equals(COMMA)) { builder.hole(parseLinearRing(stream, ignoreZValue, coerce)); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java index 79582c3365bdb..21d1bd9f25564 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import java.io.IOException; @@ -46,7 +46,7 @@ public interface ShapeParser { * if the parsers current token has been null * @throws IOException if the input could not be read */ - static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException { + static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) throws IOException { if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return null; } if (parser.currentToken() == XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java new file mode 100644 index 0000000000000..3f1e49e525e81 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -0,0 +1,336 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.DeprecatedParameters; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; + +/** + * Base class for {@link GeoShapeFieldMapper} and {@link LegacyGeoShapeFieldMapper} + */ +public abstract class BaseGeoShapeFieldMapper extends FieldMapper { + public static final String CONTENT_TYPE = "geo_shape"; + + public static class Names { + public static final ParseField ORIENTATION = new ParseField("orientation"); + public static final ParseField COERCE = new ParseField("coerce"); + } + + public static class Defaults { + public static final Explicit ORIENTATION = new Explicit<>(Orientation.RIGHT, false); + public static final Explicit COERCE = new Explicit<>(false, false); + public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); + public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + } + + public abstract static class Builder + extends FieldMapper.Builder { + protected Boolean coerce; + protected Boolean ignoreMalformed; + protected Boolean ignoreZValue; + protected Orientation orientation; + + /** default builder - used for external mapper*/ + public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType) { + super(name, fieldType, defaultFieldType); + } + + public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType, + boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) { + super(name, fieldType, defaultFieldType); + this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; + this.orientation = orientation; + this.ignoreZValue = ignoreZ; + } + + public Builder coerce(boolean coerce) { + this.coerce = coerce; + return this; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit<>(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); + } + return Defaults.COERCE; + } + + public Builder orientation(Orientation orientation) { + this.orientation = orientation; + return this; + } + + protected Explicit orientation() { + if (orientation != null) { + return new Explicit<>(orientation, true); + } + return Defaults.ORIENTATION; + } + + @Override + protected boolean defaultDocValues(Version indexCreated) { + return false; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return this; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return Defaults.IGNORE_MALFORMED; + } + + protected Explicit ignoreZValue() { + if (ignoreZValue != null) { + return new Explicit<>(ignoreZValue, true); + } + return Defaults.IGNORE_Z_VALUE; + } + + public Builder ignoreZValue(final boolean ignoreZValue) { + this.ignoreZValue = ignoreZValue; + return this; + } + + @Override + protected void setupFieldType(BuilderContext context) { + super.setupFieldType(context); + + // field mapper handles this at build time + // but prefix tree strategies require a name, so throw a similar exception + if (name().isEmpty()) { + throw new IllegalArgumentException("name cannot be empty string"); + } + + BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); + ft.setOrientation(orientation().value()); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + boolean coerce = Defaults.COERCE.value(); + boolean ignoreZ = Defaults.IGNORE_Z_VALUE.value(); + boolean ignoreMalformed = Defaults.IGNORE_MALFORMED.value(); + Orientation orientation = Defaults.ORIENTATION.value(); + DeprecatedParameters deprecatedParameters = new DeprecatedParameters(); + boolean parsedDeprecatedParams = false; + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (DeprecatedParameters.parse(name, fieldName, fieldNode, deprecatedParameters)) { + parsedDeprecatedParams = true; + iterator.remove(); + } else if (Names.ORIENTATION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + orientation = ShapeBuilder.Orientation.fromString(fieldNode.toString()); + iterator.remove(); + } else if (IGNORE_MALFORMED.equals(fieldName)) { + ignoreMalformed = XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed"); + iterator.remove(); + } else if (Names.COERCE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + coerce = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE.getPreferredName()); + iterator.remove(); + } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { + ignoreZ = XContentMapValues.nodeBooleanValue(fieldNode, + name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()); + iterator.remove(); + } + } + return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); + } + + private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + if (deprecatedParameters != null) { + return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + } + return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); + } + + private Builder getLegacyBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + return new LegacyGeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + } + } + + public abstract static class BaseGeoShapeFieldType extends MappedFieldType { + protected Orientation orientation = Defaults.ORIENTATION.value(); + + protected BaseGeoShapeFieldType() { + setIndexOptions(IndexOptions.DOCS); + setTokenized(false); + setStored(false); + setStoreTermVectors(false); + setOmitNorms(true); + } + + protected BaseGeoShapeFieldType(BaseGeoShapeFieldType ref) { + super(ref); + this.orientation = ref.orientation; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + BaseGeoShapeFieldType that = (BaseGeoShapeFieldType) o; + return orientation == that.orientation; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), orientation); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts) { + super.checkCompatibility(fieldType, conflicts); + } + + public Orientation orientation() { return this.orientation; } + + public void setOrientation(Orientation orientation) { + checkIfFrozen(); + this.orientation = orientation; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); + } + } + + protected Explicit coerce; + protected Explicit ignoreMalformed; + protected Explicit ignoreZValue; + + protected BaseGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreZValue, Settings indexSettings, + MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); + this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; + this.ignoreZValue = ignoreZValue; + } + + @Override + protected void doMerge(Mapper mergeWith) { + super.doMerge(mergeWith); + BaseGeoShapeFieldMapper gsfm = (BaseGeoShapeFieldMapper)mergeWith; + if (gsfm.coerce.explicit()) { + this.coerce = gsfm.coerce; + } + if (gsfm.ignoreMalformed.explicit()) { + this.ignoreMalformed = gsfm.ignoreMalformed; + } + if (gsfm.ignoreZValue.explicit()) { + this.ignoreZValue = gsfm.ignoreZValue; + } + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + builder.field("type", contentType()); + BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); + if (includeDefaults || ft.orientation() != Defaults.ORIENTATION.value()) { + builder.field(Names.ORIENTATION.getPreferredName(), ft.orientation()); + } + if (includeDefaults || coerce.explicit()) { + builder.field(Names.COERCE.getPreferredName(), coerce.value()); + } + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); + } + if (includeDefaults || ignoreZValue.explicit()) { + builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); + } + } + + public Explicit coerce() { + return coerce; + } + + public Explicit ignoreMalformed() { + return ignoreMalformed; + } + + public Explicit ignoreZValue() { + return ignoreZValue; + } + + public Orientation orientation() { + return ((BaseGeoShapeFieldType)fieldType).orientation(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 7de40fe337d9d..65ee2e428faa3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,48 +18,24 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.geo.Line; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.geo.SpatialStrategy; -import org.elasticsearch.common.geo.XShapeCollection; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; -import org.locationtech.spatial4j.shape.jts.JtsGeometry; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; /** - * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. + * FieldMapper for indexing {@link org.apache.lucene.document.LatLonShape}s. *

    * Currently Shapes can only be indexed and can only be queried using * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently @@ -73,554 +49,128 @@ * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] * ] * } + *

    + * or: + *

    + * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) */ -public class GeoShapeFieldMapper extends FieldMapper { - - public static final String CONTENT_TYPE = "geo_shape"; - - public static class Names { - public static final String TREE = "tree"; - public static final String TREE_GEOHASH = "geohash"; - public static final String TREE_QUADTREE = "quadtree"; - public static final String TREE_LEVELS = "tree_levels"; - public static final String TREE_PRESISION = "precision"; - public static final String DISTANCE_ERROR_PCT = "distance_error_pct"; - public static final String ORIENTATION = "orientation"; - public static final String STRATEGY = "strategy"; - public static final String STRATEGY_POINTS_ONLY = "points_only"; - public static final String COERCE = "coerce"; - } - - public static class Defaults { - public static final String TREE = Names.TREE_GEOHASH; - public static final String STRATEGY = SpatialStrategy.RECURSIVE.getStrategyName(); - public static final boolean POINTS_ONLY = false; - public static final int GEOHASH_LEVELS = GeoUtils.geoHashLevelsForPrecision("50m"); - public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision("50m"); - public static final Orientation ORIENTATION = Orientation.RIGHT; - public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d; - public static final Explicit COERCE = new Explicit<>(false, false); - public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); - public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); - - public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType(); - - static { - // setting name here is a hack so freeze can be called...instead all these options should be - // moved to the default ctor for GeoShapeFieldType, and defaultFieldType() should be removed from mappers... - FIELD_TYPE.setName("DoesNotExist"); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - } - } - - public static class Builder extends FieldMapper.Builder { - - private Boolean coerce; - private Boolean ignoreMalformed; - private Boolean ignoreZValue; +public class GeoShapeFieldMapper extends BaseGeoShapeFieldMapper { + public static class Builder extends BaseGeoShapeFieldMapper.Builder { public Builder(String name) { - super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); - } - - @Override - public GeoShapeFieldType fieldType() { - return (GeoShapeFieldType)fieldType; + super (name, new GeoShapeFieldType(), new GeoShapeFieldType()); } - public Builder coerce(boolean coerce) { - this.coerce = coerce; - return this; - } - - @Override - protected boolean defaultDocValues(Version indexCreated) { - return false; - } - - protected Explicit coerce(BuilderContext context) { - if (coerce != null) { - return new Explicit<>(coerce, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); - } - return Defaults.COERCE; - } - - public Builder ignoreMalformed(boolean ignoreMalformed) { - this.ignoreMalformed = ignoreMalformed; - return this; - } - - protected Explicit ignoreMalformed(BuilderContext context) { - if (ignoreMalformed != null) { - return new Explicit<>(ignoreMalformed, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); - } - return Defaults.IGNORE_MALFORMED; - } - - protected Explicit ignoreZValue(BuilderContext context) { - if (ignoreZValue != null) { - return new Explicit<>(ignoreZValue, true); - } - return Defaults.IGNORE_Z_VALUE; - } - - public Builder ignoreZValue(final boolean ignoreZValue) { - this.ignoreZValue = ignoreZValue; - return this; + public Builder(String name, boolean coerce, boolean ignoreMalformed, ShapeBuilder.Orientation orientation, + boolean ignoreZ) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); } @Override public GeoShapeFieldMapper build(BuilderContext context) { - GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType; - - if (geoShapeFieldType.treeLevels() == 0 && geoShapeFieldType.precisionInMeters() < 0) { - geoShapeFieldType.setDefaultDistanceErrorPct(Defaults.LEGACY_DISTANCE_ERROR_PCT); - } setupFieldType(context); - - return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), ignoreZValue(context), - context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), + ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } - public static class TypeParser implements Mapper.TypeParser { - - @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new Builder(name); - Boolean pointsOnly = null; - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (Names.TREE.equals(fieldName)) { - builder.fieldType().setTree(fieldNode.toString()); - iterator.remove(); - } else if (Names.TREE_LEVELS.equals(fieldName)) { - builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); - iterator.remove(); - } else if (Names.TREE_PRESISION.equals(fieldName)) { - builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), - DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); - iterator.remove(); - } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { - builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); - iterator.remove(); - } else if (Names.ORIENTATION.equals(fieldName)) { - builder.fieldType().setOrientation(ShapeBuilder.Orientation.fromString(fieldNode.toString())); - iterator.remove(); - } else if (Names.STRATEGY.equals(fieldName)) { - builder.fieldType().setStrategyName(fieldNode.toString()); - iterator.remove(); - } else if (IGNORE_MALFORMED.equals(fieldName)) { - builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed")); - iterator.remove(); - } else if (Names.COERCE.equals(fieldName)) { - builder.coerce(XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE)); - iterator.remove(); - } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { - builder.ignoreZValue(XContentMapValues.nodeBooleanValue(fieldNode, - name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); - iterator.remove(); - } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) { - pointsOnly = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.STRATEGY_POINTS_ONLY); - iterator.remove(); - } - } - if (pointsOnly != null) { - if (builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) && pointsOnly == false) { - throw new IllegalArgumentException("points_only cannot be set to false for term strategy"); - } else { - builder.fieldType().setPointsOnly(pointsOnly); - } - } - return builder; + public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { + public GeoShapeFieldType() { + super(); } - } - - public static final class GeoShapeFieldType extends MappedFieldType { - - private String tree = Defaults.TREE; - private String strategyName = Defaults.STRATEGY; - private boolean pointsOnly = Defaults.POINTS_ONLY; - private int treeLevels = 0; - private double precisionInMeters = -1; - private Double distanceErrorPct; - private double defaultDistanceErrorPct = 0.0; - private Orientation orientation = Defaults.ORIENTATION; - - // these are built when the field type is frozen - private PrefixTreeStrategy defaultStrategy; - private RecursivePrefixTreeStrategy recursiveStrategy; - private TermQueryPrefixTreeStrategy termStrategy; - - public GeoShapeFieldType() {} protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); - this.tree = ref.tree; - this.strategyName = ref.strategyName; - this.pointsOnly = ref.pointsOnly; - this.treeLevels = ref.treeLevels; - this.precisionInMeters = ref.precisionInMeters; - this.distanceErrorPct = ref.distanceErrorPct; - this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; - this.orientation = ref.orientation; } @Override public GeoShapeFieldType clone() { return new GeoShapeFieldType(this); } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - GeoShapeFieldType that = (GeoShapeFieldType) o; - return treeLevels == that.treeLevels && - precisionInMeters == that.precisionInMeters && - defaultDistanceErrorPct == that.defaultDistanceErrorPct && - Objects.equals(tree, that.tree) && - Objects.equals(strategyName, that.strategyName) && - pointsOnly == that.pointsOnly && - Objects.equals(distanceErrorPct, that.distanceErrorPct) && - orientation == that.orientation; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), tree, strategyName, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, - defaultDistanceErrorPct, orientation); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public void freeze() { - super.freeze(); - // This is a bit hackish: we need to setup the spatial tree and strategies once the field name is set, which - // must be by the time freeze is called. - SpatialPrefixTree prefixTree; - if ("geohash".equals(tree)) { - prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true)); - } else if ("legacyquadtree".equals(tree)) { - prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else if ("quadtree".equals(tree)) { - prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + tree + "]"); - } - - recursiveStrategy = new RecursivePrefixTreeStrategy(prefixTree, name()); - recursiveStrategy.setDistErrPct(distanceErrorPct()); - recursiveStrategy.setPruneLeafyBranches(false); - termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); - termStrategy.setDistErrPct(distanceErrorPct()); - defaultStrategy = resolveStrategy(strategyName); - defaultStrategy.setPointsOnly(pointsOnly); - } - - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts) { - super.checkCompatibility(fieldType, conflicts); - GeoShapeFieldType other = (GeoShapeFieldType)fieldType; - // prevent user from changing strategies - if (strategyName().equals(other.strategyName()) == false) { - conflicts.add("mapper [" + name() + "] has different [strategy]"); - } - - // prevent user from changing trees (changes encoding) - if (tree().equals(other.tree()) == false) { - conflicts.add("mapper [" + name() + "] has different [tree]"); - } - - if ((pointsOnly() != other.pointsOnly())) { - conflicts.add("mapper [" + name() + "] has different points_only"); - } - - // TODO we should allow this, but at the moment levels is used to build bookkeeping variables - // in lucene's SpatialPrefixTree implementations, need a patch to correct that first - if (treeLevels() != other.treeLevels()) { - conflicts.add("mapper [" + name() + "] has different [tree_levels]"); - } - if (precisionInMeters() != other.precisionInMeters()) { - conflicts.add("mapper [" + name() + "] has different [precision]"); - } - } - - private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { - if (treeLevels > 0 || precisionInMeters >= 0) { - return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) - : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); - } - return defaultLevels; - } - - public String tree() { - return tree; - } - - public void setTree(String tree) { - checkIfFrozen(); - this.tree = tree; - } - - public String strategyName() { - return strategyName; - } - - public void setStrategyName(String strategyName) { - checkIfFrozen(); - this.strategyName = strategyName; - if (this.strategyName.equals(SpatialStrategy.TERM.getStrategyName())) { - this.pointsOnly = true; - } - } - - public boolean pointsOnly() { - return pointsOnly; - } - - public void setPointsOnly(boolean pointsOnly) { - checkIfFrozen(); - this.pointsOnly = pointsOnly; - } - public int treeLevels() { - return treeLevels; - } - - public void setTreeLevels(int treeLevels) { - checkIfFrozen(); - this.treeLevels = treeLevels; - } - - public double precisionInMeters() { - return precisionInMeters; - } - - public void setPrecisionInMeters(double precisionInMeters) { - checkIfFrozen(); - this.precisionInMeters = precisionInMeters; - } - - public double distanceErrorPct() { - return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; - } - - public void setDistanceErrorPct(double distanceErrorPct) { - checkIfFrozen(); - this.distanceErrorPct = distanceErrorPct; - } - - public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { - checkIfFrozen(); - this.defaultDistanceErrorPct = defaultDistanceErrorPct; - } - - public Orientation orientation() { return this.orientation; } - - public void setOrientation(Orientation orientation) { - checkIfFrozen(); - this.orientation = orientation; - } - - public PrefixTreeStrategy defaultStrategy() { - return this.defaultStrategy; - } - - public PrefixTreeStrategy resolveStrategy(SpatialStrategy strategy) { - return resolveStrategy(strategy.getStrategyName()); - } - - public PrefixTreeStrategy resolveStrategy(String strategyName) { - if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { - return recursiveStrategy; - } - if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { - return termStrategy; - } - throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); - } - - @Override - public Query existsQuery(QueryShardContext context) { - return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); - } } - protected Explicit coerce; - protected Explicit ignoreMalformed; - protected Explicit ignoreZValue; - - public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit ignoreMalformed, - Explicit coerce, Explicit ignoreZValue, Settings indexSettings, + public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.ignoreZValue = ignoreZValue; + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + multiFields, copyTo); } @Override public GeoShapeFieldType fieldType() { return (GeoShapeFieldType) super.fieldType(); } + + /** parsing logic for {@link LatLonShape} indexing */ @Override public void parse(ParseContext context) throws IOException { try { - Shape shape = context.parseExternalValue(Shape.class); + Object shape = context.parseExternalValue(Object.class); if (shape == null) { ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); if (shapeBuilder == null) { return; } - shape = shapeBuilder.buildS4J(); - } - if (fieldType().pointsOnly() == true) { - // index configured for pointsOnly - if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { - // MULTIPOINT data: index each point separately - List shapes = ((XShapeCollection) shape).getShapes(); - for (Shape s : shapes) { - indexShape(context, s); - } - return; - } else if (shape instanceof Point == false) { - throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " - + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) - + " was found"); - } + shape = shapeBuilder.buildLucene(); } indexShape(context, shape); } catch (Exception e) { if (ignoreMalformed.value() == false) { throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), - fieldType().typeName()); - } - context.addIgnoredField(fieldType.name()); - } - } - - private void indexShape(ParseContext context, Shape shape) { - List fields = new ArrayList<>(Arrays.asList(fieldType().defaultStrategy().createIndexableFields(shape))); - createFieldNamesField(context, fields); - for (IndexableField field : fields) { - context.doc().add(field); - } - } - - @Override - protected void parseCreateField(ParseContext context, List fields) throws IOException { - } - - @Override - protected void doMerge(Mapper mergeWith) { - super.doMerge(mergeWith); - - GeoShapeFieldMapper gsfm = (GeoShapeFieldMapper)mergeWith; - if (gsfm.coerce.explicit()) { - this.coerce = gsfm.coerce; - } - if (gsfm.ignoreMalformed.explicit()) { - this.ignoreMalformed = gsfm.ignoreMalformed; - } - if (gsfm.ignoreZValue.explicit()) { - this.ignoreZValue = gsfm.ignoreZValue; - } - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - builder.field("type", contentType()); - - if (includeDefaults || fieldType().tree().equals(Defaults.TREE) == false) { - builder.field(Names.TREE, fieldType().tree()); - } - - if (fieldType().treeLevels() != 0) { - builder.field(Names.TREE_LEVELS, fieldType().treeLevels()); - } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified - if ("geohash".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.GEOHASH_LEVELS); - } else if ("legacyquadtree".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); - } else if ("quadtree".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); - } - } - if (fieldType().precisionInMeters() != -1) { - builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(fieldType().precisionInMeters())); - } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified - builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(50)); - } - if (includeDefaults || fieldType().strategyName().equals(Defaults.STRATEGY) == false) { - builder.field(Names.STRATEGY, fieldType().strategyName()); - } - if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { - builder.field(Names.DISTANCE_ERROR_PCT, fieldType().distanceErrorPct()); - } - if (includeDefaults || fieldType().orientation() != Defaults.ORIENTATION) { - builder.field(Names.ORIENTATION, fieldType().orientation()); - } - if (fieldType().strategyName().equals(SpatialStrategy.TERM.getStrategyName())) { - // For TERMs strategy the defaults for points only change to true - if (includeDefaults || fieldType().pointsOnly() != true) { - builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); + fieldType().typeName()); + } + context.addIgnoredField(fieldType().name()); + } + } + + private void indexShape(ParseContext context, Object luceneShape) { + if (luceneShape instanceof GeoPoint) { + GeoPoint pt = (GeoPoint) luceneShape; + indexFields(context, LatLonShape.createIndexableFields(name(), pt.lat(), pt.lon())); + } else if (luceneShape instanceof double[]) { + double[] pt = (double[]) luceneShape; + indexFields(context, LatLonShape.createIndexableFields(name(), pt[1], pt[0])); + } else if (luceneShape instanceof Line) { + indexFields(context, LatLonShape.createIndexableFields(name(), (Line)luceneShape)); + } else if (luceneShape instanceof Polygon) { + indexFields(context, LatLonShape.createIndexableFields(name(), (Polygon) luceneShape)); + } else if (luceneShape instanceof double[][]) { + double[][] pts = (double[][])luceneShape; + for (int i = 0; i < pts.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), pts[i][1], pts[i][0])); + } + } else if (luceneShape instanceof Line[]) { + Line[] lines = (Line[]) luceneShape; + for (int i = 0; i < lines.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), lines[i])); + } + } else if (luceneShape instanceof Polygon[]) { + Polygon[] polys = (Polygon[]) luceneShape; + for (int i = 0; i < polys.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), polys[i])); + } + } else if (luceneShape instanceof Rectangle) { + // index rectangle as a polygon + Rectangle r = (Rectangle) luceneShape; + Polygon p = new Polygon(new double[]{r.minLat, r.minLat, r.maxLat, r.maxLat, r.minLat}, + new double[]{r.minLon, r.maxLon, r.maxLon, r.minLon, r.minLon}); + indexFields(context, LatLonShape.createIndexableFields(name(), p)); + } else if (luceneShape instanceof Object[]) { + // recurse to index geometry collection + for (Object o : (Object[])luceneShape) { + indexShape(context, o); } } else { - if (includeDefaults || fieldType().pointsOnly() != GeoShapeFieldMapper.Defaults.POINTS_ONLY) { - builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); - } - } - if (includeDefaults || coerce.explicit()) { - builder.field(Names.COERCE, coerce.value()); + throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass() + "] while indexing shape"); } - if (includeDefaults || ignoreMalformed.explicit()) { - builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); - } - if (includeDefaults || ignoreZValue.explicit()) { - builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); - } - } - - public Explicit coerce() { - return coerce; - } - - public Explicit ignoreMalformed() { - return ignoreMalformed; - } - - public Explicit ignoreZValue() { - return ignoreZValue; } - @Override - protected String contentType() { - return CONTENT_TYPE; + private void indexFields(ParseContext context, Field[] fields) { + ArrayList flist = new ArrayList<>(Arrays.asList(fields)); + createFieldNamesField(context, flist); + for (IndexableField f : flist) { + context.doc().add(f); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java new file mode 100644 index 0000000000000..b68e48305b24b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -0,0 +1,596 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.ShapesAvailability; +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.XShapeCollection; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.jts.JtsGeometry; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. + *

    + * Currently Shapes can only be indexed and can only be queried using + * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently + * a lot of behavior in this Mapper is disabled. + *

    + * Format supported: + *

    + * "field" : { + * "type" : "polygon", + * "coordinates" : [ + * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] + * ] + * } + *

    + * or: + *

    + * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) + * + * @deprecated use {@link GeoShapeFieldMapper} + */ +@Deprecated +public class LegacyGeoShapeFieldMapper extends BaseGeoShapeFieldMapper { + + public static final String CONTENT_TYPE = "geo_shape"; + + @Deprecated + public static class DeprecatedParameters { + public static class Names { + public static final ParseField STRATEGY = new ParseField("strategy"); + public static final ParseField TREE = new ParseField("tree"); + public static final ParseField TREE_LEVELS = new ParseField("tree_levels"); + public static final ParseField PRECISION = new ParseField("precision"); + public static final ParseField DISTANCE_ERROR_PCT = new ParseField("distance_error_pct"); + public static final ParseField POINTS_ONLY = new ParseField("points_only"); + } + + public static class PrefixTrees { + public static final String LEGACY_QUADTREE = "legacyquadtree"; + public static final String QUADTREE = "quadtree"; + public static final String GEOHASH = "geohash"; + } + + public static class Defaults { + public static final SpatialStrategy STRATEGY = SpatialStrategy.RECURSIVE; + public static final String TREE = "quadtree"; + public static final String PRECISION = "50m"; + public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision(PRECISION); + public static final int GEOHASH_TREE_LEVELS = GeoUtils.geoHashLevelsForPrecision(PRECISION); + public static final boolean POINTS_ONLY = false; + public static final double DISTANCE_ERROR_PCT = 0.025d; + } + + public SpatialStrategy strategy = null; + public String tree = null; + public int treeLevels = Integer.MIN_VALUE; + public String precision = null; + public Boolean pointsOnly = null; + public double distanceErrorPct = Double.NaN; + + public void setSpatialStrategy(SpatialStrategy strategy) { + this.strategy = strategy; + } + + public void setTree(String prefixTree) { + this.tree = prefixTree; + } + + public void setTreeLevels(int treeLevels) { + this.treeLevels = treeLevels; + } + + public void setPrecision(String precision) { + this.precision = precision; + } + + public void setPointsOnly(boolean pointsOnly) { + if (this.strategy == SpatialStrategy.TERM && pointsOnly == false) { + throw new ElasticsearchParseException("points_only cannot be set to false for term strategy"); + } + this.pointsOnly = pointsOnly; + } + + public void setDistanceErrorPct(double distanceErrorPct) { + this.distanceErrorPct = distanceErrorPct; + } + + protected void setup() { + if (strategy == null) { + strategy = Defaults.STRATEGY; + } + if (tree == null) { + tree = Defaults.TREE; + } + if (Double.isNaN(distanceErrorPct)) { + if (precision != null || treeLevels != Integer.MIN_VALUE) { + distanceErrorPct = 0d; + } else { + distanceErrorPct = Defaults.DISTANCE_ERROR_PCT; + } + } + if (treeLevels == Integer.MIN_VALUE && precision == null) { + // set default precision if treeLevels is not explicitly set + precision = Defaults.PRECISION; + } + if (treeLevels == Integer.MIN_VALUE) { + if (precision.equals(Defaults.PRECISION)) { + treeLevels = tree.equals(Defaults.TREE) + ? Defaults.QUADTREE_LEVELS + : Defaults.GEOHASH_TREE_LEVELS; + } else { + treeLevels = tree == Defaults.TREE + ? GeoUtils.quadTreeLevelsForPrecision(precision) + : GeoUtils.geoHashLevelsForPrecision(precision); + } + } + if (pointsOnly == null) { + if (strategy == SpatialStrategy.TERM) { + pointsOnly = true; + } else { + pointsOnly = Defaults.POINTS_ONLY; + } + } + } + + public static boolean parse(String name, String fieldName, Object fieldNode, DeprecatedParameters deprecatedParameters) { + if (Names.STRATEGY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setSpatialStrategy(SpatialStrategy.fromString(fieldNode.toString())); + } else if (Names.TREE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setTree(fieldNode.toString()); + } else if (Names.TREE_LEVELS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setTreeLevels(Integer.parseInt(fieldNode.toString())); + } else if (Names.PRECISION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setPrecision(fieldNode.toString()); + } else if (Names.DISTANCE_ERROR_PCT.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); + } else if (Names.POINTS_ONLY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setPointsOnly( + XContentMapValues.nodeBooleanValue(fieldNode, name + "." + DeprecatedParameters.Names.POINTS_ONLY)); + } else { + return false; + } + return true; + } + + private static void checkPrefixTreeSupport(String fieldName) { + if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { + throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", + fieldName, CONTENT_TYPE); + } + DEPRECATION_LOGGER.deprecated("Field parameter [{}] is deprecated and will be removed in a future version.", + fieldName); + } + } + + private static final Logger logger = LogManager.getLogger(LegacyGeoShapeFieldMapper.class); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); + + public static class Builder extends BaseGeoShapeFieldMapper.Builder { + + DeprecatedParameters deprecatedParameters; + + public Builder(String name) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType()); + this.deprecatedParameters = new DeprecatedParameters(); + this.deprecatedParameters.setup(); + } + + public Builder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); + this.deprecatedParameters = deprecatedParameters; + this.deprecatedParameters.setup(); + } + + @Override + public GeoShapeFieldType fieldType() { + return (GeoShapeFieldType)fieldType; + } + + private void setupFieldTypeDeprecatedParameters() { + GeoShapeFieldType ft = fieldType(); + ft.setStrategy(deprecatedParameters.strategy); + ft.setTree(deprecatedParameters.tree); + ft.setTreeLevels(deprecatedParameters.treeLevels); + if (deprecatedParameters.precision != null) { + // precision is only set iff: a. treeLevel is not explicitly set, b. its explicitly set + ft.setPrecisionInMeters(DistanceUnit.parse(deprecatedParameters.precision, + DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); + } + ft.setDistanceErrorPct(deprecatedParameters.distanceErrorPct); + ft.setPointsOnly(deprecatedParameters.pointsOnly); + } + + private void setupPrefixTrees() { + GeoShapeFieldType ft = fieldType(); + SpatialPrefixTree prefixTree; + if (ft.tree().equals(DeprecatedParameters.PrefixTrees.GEOHASH)) { + prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS, true)); + } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE)) { + prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); + } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.QUADTREE)) { + prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + ft.tree() + "]"); + } + + // setup prefix trees regardless of strategy (this is used for the QueryBuilder) + // recursive: + RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, ft.name()); + rpts.setDistErrPct(ft.distanceErrorPct()); + rpts.setPruneLeafyBranches(false); + ft.recursiveStrategy = rpts; + + // term: + TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, ft.name()); + termStrategy.setDistErrPct(ft.distanceErrorPct()); + ft.termStrategy = termStrategy; + + // set default (based on strategy): + ft.defaultPrefixTreeStrategy = ft.resolvePrefixTreeStrategy(ft.strategy()); + ft.defaultPrefixTreeStrategy.setPointsOnly(ft.pointsOnly()); + } + + @Override + protected void setupFieldType(BuilderContext context) { + super.setupFieldType(context); + + // field mapper handles this at build time + // but prefix tree strategies require a name, so throw a similar exception + if (fieldType().name().isEmpty()) { + throw new IllegalArgumentException("name cannot be empty string"); + } + + // setup the deprecated parameters and the prefix tree configuration + setupFieldTypeDeprecatedParameters(); + setupPrefixTrees(); + } + + private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { + if (treeLevels > 0 || precisionInMeters >= 0) { + return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) + : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); + } + return defaultLevels; + } + + @Override + public LegacyGeoShapeFieldMapper build(BuilderContext context) { + setupFieldType(context); + + return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), + coerce(context), orientation(), ignoreZValue(), context.indexSettings(), + multiFieldsBuilder.build(this, context), copyTo); + } + } + + public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { + + private String tree = DeprecatedParameters.Defaults.TREE; + private SpatialStrategy strategy = DeprecatedParameters.Defaults.STRATEGY; + private boolean pointsOnly = DeprecatedParameters.Defaults.POINTS_ONLY; + private int treeLevels = 0; + private double precisionInMeters = -1; + private Double distanceErrorPct; + private double defaultDistanceErrorPct = 0.0; + + // these are built when the field type is frozen + private PrefixTreeStrategy defaultPrefixTreeStrategy; + private RecursivePrefixTreeStrategy recursiveStrategy; + private TermQueryPrefixTreeStrategy termStrategy; + + public GeoShapeFieldType() { + setIndexOptions(IndexOptions.DOCS); + setTokenized(false); + setStored(false); + setStoreTermVectors(false); + setOmitNorms(true); + } + + protected GeoShapeFieldType(GeoShapeFieldType ref) { + super(ref); + this.tree = ref.tree; + this.strategy = ref.strategy; + this.pointsOnly = ref.pointsOnly; + this.treeLevels = ref.treeLevels; + this.precisionInMeters = ref.precisionInMeters; + this.distanceErrorPct = ref.distanceErrorPct; + this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; + } + + @Override + public GeoShapeFieldType clone() { + return new GeoShapeFieldType(this); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + GeoShapeFieldType that = (GeoShapeFieldType) o; + return treeLevels == that.treeLevels && + precisionInMeters == that.precisionInMeters && + defaultDistanceErrorPct == that.defaultDistanceErrorPct && + Objects.equals(tree, that.tree) && + Objects.equals(strategy, that.strategy) && + pointsOnly == that.pointsOnly && + Objects.equals(distanceErrorPct, that.distanceErrorPct); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, + defaultDistanceErrorPct); + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts) { + super.checkCompatibility(fieldType, conflicts); + GeoShapeFieldType other = (GeoShapeFieldType)fieldType; + // prevent user from changing strategies + if (strategy() != other.strategy()) { + conflicts.add("mapper [" + name() + "] has different [strategy]"); + } + + // prevent user from changing trees (changes encoding) + if (tree().equals(other.tree()) == false) { + conflicts.add("mapper [" + name() + "] has different [tree]"); + } + + if ((pointsOnly() != other.pointsOnly())) { + conflicts.add("mapper [" + name() + "] has different points_only"); + } + + // TODO we should allow this, but at the moment levels is used to build bookkeeping variables + // in lucene's SpatialPrefixTree implementations, need a patch to correct that first + if (treeLevels() != other.treeLevels()) { + conflicts.add("mapper [" + name() + "] has different [tree_levels]"); + } + if (precisionInMeters() != other.precisionInMeters()) { + conflicts.add("mapper [" + name() + "] has different [precision]"); + } + } + + public String tree() { + return tree; + } + + public void setTree(String tree) { + checkIfFrozen(); + this.tree = tree; + } + + public SpatialStrategy strategy() { + return strategy; + } + + public void setStrategy(SpatialStrategy strategy) { + checkIfFrozen(); + this.strategy = strategy; + if (this.strategy.equals(SpatialStrategy.TERM)) { + this.pointsOnly = true; + } + } + + public boolean pointsOnly() { + return pointsOnly; + } + + public void setPointsOnly(boolean pointsOnly) { + checkIfFrozen(); + this.pointsOnly = pointsOnly; + } + public int treeLevels() { + return treeLevels; + } + + public void setTreeLevels(int treeLevels) { + checkIfFrozen(); + this.treeLevels = treeLevels; + } + + public double precisionInMeters() { + return precisionInMeters; + } + + public void setPrecisionInMeters(double precisionInMeters) { + checkIfFrozen(); + this.precisionInMeters = precisionInMeters; + } + + public double distanceErrorPct() { + return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; + } + + public void setDistanceErrorPct(double distanceErrorPct) { + checkIfFrozen(); + this.distanceErrorPct = distanceErrorPct; + } + + public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { + checkIfFrozen(); + this.defaultDistanceErrorPct = defaultDistanceErrorPct; + } + + public PrefixTreeStrategy defaultPrefixTreeStrategy() { + return this.defaultPrefixTreeStrategy; + } + + public PrefixTreeStrategy resolvePrefixTreeStrategy(SpatialStrategy strategy) { + return resolvePrefixTreeStrategy(strategy.getStrategyName()); + } + + public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { + if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { + return recursiveStrategy; + } + if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { + return termStrategy; + } + throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); + } + } + + public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, Explicit orientation, + Explicit ignoreZValue, Settings indexSettings, + MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + multiFields, copyTo); + } + + @Override + public GeoShapeFieldType fieldType() { + return (GeoShapeFieldType) super.fieldType(); + } + + @Override + public void parse(ParseContext context) throws IOException { + try { + Shape shape = context.parseExternalValue(Shape.class); + if (shape == null) { + ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); + if (shapeBuilder == null) { + return; + } + shape = shapeBuilder.buildS4J(); + } + if (fieldType().pointsOnly() == true) { + // index configured for pointsOnly + if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { + // MULTIPOINT data: index each point separately + List shapes = ((XShapeCollection) shape).getShapes(); + for (Shape s : shapes) { + indexShape(context, s); + } + return; + } else if (shape instanceof Point == false) { + throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " + + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + + " was found"); + } + } + indexShape(context, shape); + } catch (Exception e) { + if (ignoreMalformed.value() == false) { + throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), + fieldType().typeName()); + } + context.addIgnoredField(fieldType.name()); + } + } + + private void indexShape(ParseContext context, Shape shape) { + List fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape))); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || fieldType().tree().equals(DeprecatedParameters.Defaults.TREE) == false) { + builder.field(DeprecatedParameters.Names.TREE.getPreferredName(), fieldType().tree()); + } + + if (fieldType().treeLevels() != 0) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), fieldType().treeLevels()); + } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified + if (DeprecatedParameters.PrefixTrees.GEOHASH.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS); + } else if (DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.QUADTREE_LEVELS); + } else if (DeprecatedParameters.PrefixTrees.QUADTREE.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.QUADTREE_LEVELS); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); + } + } + if (fieldType().precisionInMeters() != -1) { + builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), + DistanceUnit.METERS.toString(fieldType().precisionInMeters())); + } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified + builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), + DistanceUnit.METERS.toString(50)); + } + + builder.field(DeprecatedParameters.Names.STRATEGY.getPreferredName(), fieldType().strategy().getStrategyName()); + + if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { + builder.field(DeprecatedParameters.Names.DISTANCE_ERROR_PCT.getPreferredName(), fieldType().distanceErrorPct()); + } + if (fieldType().strategy() == SpatialStrategy.TERM) { + // For TERMs strategy the defaults for points only change to true + if (includeDefaults || fieldType().pointsOnly() != true) { + builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); + } + } else { + if (includeDefaults || fieldType().pointsOnly() != DeprecatedParameters.Defaults.POINTS_ONLY) { + builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index c517050896946..6ee0f3f10ddcc 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -19,6 +19,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.geo.Line; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -36,8 +40,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.parsers.ShapeParser; @@ -48,7 +53,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; @@ -329,9 +335,9 @@ public GeoShapeQueryBuilder relation(ShapeRelation relation) { if (relation == null) { throw new IllegalArgumentException("No Shape Relation defined"); } - if (strategy != null && strategy == SpatialStrategy.TERM && relation != ShapeRelation.INTERSECTS) { + if (SpatialStrategy.TERM.equals(strategy) && relation != ShapeRelation.INTERSECTS) { throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" - + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); + + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); } this.relation = relation; return this; @@ -376,34 +382,98 @@ protected Query doToQuery(QueryShardContext context) { } else { throw new QueryShardException(context, "failed to find geo_shape field [" + fieldName + "]"); } - } else if (fieldType.typeName().equals(GeoShapeFieldMapper.CONTENT_TYPE) == false) { + } else if (fieldType.typeName().equals(BaseGeoShapeFieldMapper.CONTENT_TYPE) == false) { throw new QueryShardException(context, "Field [" + fieldName + "] is not of type [geo_shape] but of type [" + fieldType.typeName() + "]"); } - final GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType; - - PrefixTreeStrategy strategy = shapeFieldType.defaultStrategy(); - if (this.strategy != null) { - strategy = shapeFieldType.resolveStrategy(this.strategy); - } + final BaseGeoShapeFieldMapper.BaseGeoShapeFieldType ft = (BaseGeoShapeFieldMapper.BaseGeoShapeFieldType) fieldType; Query query; - if (strategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { - // this strategy doesn't support disjoint anymore: but it did - // before, including creating lucene fieldcache (!) - // in this case, execute disjoint as exists && !intersects - BooleanQuery.Builder bool = new BooleanQuery.Builder(); - Query exists = ExistsQueryBuilder.newFilter(context, fieldName); - Query intersects = strategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); - bool.add(exists, BooleanClause.Occur.MUST); - bool.add(intersects, BooleanClause.Occur.MUST_NOT); - query = new ConstantScoreQuery(bool.build()); + if (strategy != null || ft instanceof LegacyGeoShapeFieldMapper.GeoShapeFieldType) { + LegacyGeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (LegacyGeoShapeFieldMapper.GeoShapeFieldType) ft; + SpatialStrategy spatialStrategy = shapeFieldType.strategy(); + if (this.strategy != null) { + spatialStrategy = this.strategy; + } + PrefixTreeStrategy prefixTreeStrategy = shapeFieldType.resolvePrefixTreeStrategy(spatialStrategy); + if (prefixTreeStrategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { + // this strategy doesn't support disjoint anymore: but it did + // before, including creating lucene fieldcache (!) + // in this case, execute disjoint as exists && !intersects + BooleanQuery.Builder bool = new BooleanQuery.Builder(); + Query exists = ExistsQueryBuilder.newFilter(context, fieldName); + Query intersects = prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); + bool.add(exists, BooleanClause.Occur.MUST); + bool.add(intersects, BooleanClause.Occur.MUST_NOT); + query = new ConstantScoreQuery(bool.build()); + } else { + query = new ConstantScoreQuery(prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, relation))); + } } else { - query = new ConstantScoreQuery(strategy.makeQuery(getArgs(shapeToQuery, relation))); + query = new ConstantScoreQuery(getVectorQuery(context, shapeToQuery)); } return query; } + private Query getVectorQuery(QueryShardContext context, ShapeBuilder queryShapeBuilder) { + // CONTAINS queries are not yet supported by VECTOR strategy + if (relation == ShapeRelation.CONTAINS) { + throw new QueryShardException(context, + ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "]"); + } + + // wrap geoQuery as a ConstantScoreQuery + return getVectorQueryFromShape(context, queryShapeBuilder.buildLucene()); + } + + private Query getVectorQueryFromShape(QueryShardContext context, Object queryShape) { + Query geoQuery; + if (queryShape instanceof Line[]) { + geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line[]) queryShape); + } else if (queryShape instanceof Polygon[]) { + geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon[]) queryShape); + } else if (queryShape instanceof Line) { + geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line) queryShape); + } else if (queryShape instanceof Polygon) { + geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon) queryShape); + } else if (queryShape instanceof Rectangle) { + Rectangle r = (Rectangle) queryShape; + geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), + r.minLat, r.maxLat, r.minLon, r.maxLon); + } else if (queryShape instanceof double[][]) { + // note: we decompose point queries into a bounding box query with min values == max values + // to do this for multipoint we would have to create a BooleanQuery for each point + // this is *way* too costly. So we do not allow multipoint queries + throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.MULTIPOINT + " queries"); + } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { + // for now just create a single bounding box query with min values == max values + double[] pt; + if (queryShape instanceof GeoPoint) { + pt = new double[] {((GeoPoint)queryShape).lon(), ((GeoPoint)queryShape).lat()}; + } else { + pt = (double[])queryShape; + if (pt.length != 2) { + throw new QueryShardException(context, "Expected double array of length 2. " + + "But found length " + pt.length + " for field [" + fieldName + "]"); + } + } + return LatLonShape.newBoxQuery(fieldName, relation.getLuceneRelation(), pt[1], pt[1], pt[0], pt[0]); + } else if (queryShape instanceof Object[]) { + geoQuery = createGeometryCollectionQuery(context, (Object[]) queryShape); + } else { + throw new QueryShardException(context, "Field [" + fieldName + "] found and unknown shape"); + } + return geoQuery; + } + + private Query createGeometryCollectionQuery(QueryShardContext context, Object... shapes) { + BooleanQuery.Builder bqb = new BooleanQuery.Builder(); + for (Object shape : shapes) { + bqb.add(getVectorQueryFromShape(context, shape), BooleanClause.Occur.SHOULD); + } + return bqb.build(); + } + /** * Fetches the Shape with the given ID in the given type and index. * @@ -414,9 +484,6 @@ protected Query doToQuery(QueryShardContext context) { * Shape itself is located */ private void fetch(Client client, GetRequest getRequest, String path, ActionListener listener) { - if (ShapesAvailability.JTS_AVAILABLE == false) { - throw new IllegalStateException("JTS not available"); - } getRequest.preference("_local"); client.get(getRequest, new ActionListener(){ diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index a1038853c0670..24b5d7f427ca2 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -25,13 +25,13 @@ import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; @@ -39,7 +39,6 @@ import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -132,10 +131,7 @@ private Map getMappers(List mapperPlugi mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - - if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { - mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); - } + mappers.put(BaseGeoShapeFieldMapper.CONTENT_TYPE, new BaseGeoShapeFieldMapper.TypeParser()); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index a9a210549064f..2acabee8797f4 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions; @@ -296,7 +296,8 @@ public void testParse3DPolygon() throws IOException { LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper) (new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); @@ -896,7 +897,6 @@ public void testParseMultiPoint() throws IOException { .startArray().value(101.0).value(1.0).endArray() .endArray() .endObject(); - ShapeCollection expected = shapeCollection( SPATIAL_CONTEXT.makePoint(100, 0), SPATIAL_CONTEXT.makePoint(101, 1.0)); @@ -968,7 +968,6 @@ public void testParseMultiPolygon() throws IOException { shellCoordinates.add(new Coordinate(102, 2)); shellCoordinates.add(new Coordinate(102, 3)); - shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); Polygon withoutHoles = GEOMETRY_FACTORY.createPolygon(shell, null); @@ -1149,7 +1148,6 @@ public void testThatParserExtractsCorrectTypeAndCoordinatesFromArbitraryJson() t .startObject("nested").startArray("coordinates").value(200.0).value(0.0).endArray().endObject() .startObject("lala").field("type", "NotAPoint").endObject() .endObject(); - Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson, true); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 1b4c0b9dce048..94c96e00d9236 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.locationtech.jts.geom.Coordinate; @@ -146,7 +147,6 @@ private List randomLineStringCoords() { @Override public void testParseLineString() throws IOException { List coordinates = randomLineStringCoords(); - LineString expected = GEOMETRY_FACTORY.createLineString(coordinates.toArray(new Coordinate[coordinates.size()])); assertExpected(jtsGeom(expected), new LineStringBuilder(coordinates), true); @@ -279,13 +279,14 @@ public void testParseMixedDimensionPolyWithHole() throws IOException { parser.nextToken(); Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_7_0_0) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext); + final GeoShapeFieldMapper mapperBuilder = + (GeoShapeFieldMapper) (new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext)); // test store z disabled ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, @@ -323,7 +324,8 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); // test store z disabled ElasticsearchException e = expectThrows(ElasticsearchException.class, @@ -352,7 +354,8 @@ public void testParsePolyWithStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, mapperBuilder); assertEquals(shapeBuilder.numDimensions(), 3); @@ -372,12 +375,14 @@ public void testParseOpenPolygon() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper defaultMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper defaultMapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext)); ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> ShapeParser.parse(parser, defaultMapperBuilder)); assertEquals("invalid LinearRing found (coordinates are not closed)", exception.getMessage()); - final GeoShapeFieldMapper coercingMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper coercingMapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext)); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, coercingMapperBuilder); assertNotNull(shapeBuilder); assertEquals("polygon ((100.0 5.0, 100.0 10.0, 90.0 10.0, 90.0 5.0, 100.0 5.0))", shapeBuilder.toWKT()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 0e6854c41e3c4..20c49c00935e3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -24,8 +24,8 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; import org.elasticsearch.common.geo.builders.PointBuilder; -import org.locationtech.spatial4j.shape.Point; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; @@ -63,6 +63,7 @@ public static class Builder extends FieldMapper.Builder private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); private GeoPointFieldMapper.Builder latLonPointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT); private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); + private LegacyGeoShapeFieldMapper.Builder legacyShapeBuilder = new LegacyGeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); private Mapper.Builder stringBuilder; private String generatedValue; private String mapperName; @@ -86,7 +87,9 @@ public ExternalMapper build(BuilderContext context) { BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = latLonPointBuilder.build(context); - GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context); + BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_6_0)) + ? legacyShapeBuilder.build(context) + : shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); context.path().remove(); @@ -150,13 +153,13 @@ public Query existsQuery(QueryShardContext context) { private BinaryFieldMapper binMapper; private BooleanFieldMapper boolMapper; private GeoPointFieldMapper pointMapper; - private GeoShapeFieldMapper shapeMapper; + private BaseGeoShapeFieldMapper shapeMapper; private FieldMapper stringMapper; public ExternalMapper(String simpleName, MappedFieldType fieldType, String generatedValue, String mapperName, BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, - GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, + BaseGeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, new ExternalFieldType(), indexSettings, multiFields, copyTo); this.generatedValue = generatedValue; @@ -182,8 +185,12 @@ public void parse(ParseContext context) throws IOException { pointMapper.parse(context.createExternalValueContext(point)); // Let's add a Dummy Shape - Point shape = new PointBuilder(-100, 45).buildS4J(); - shapeMapper.parse(context.createExternalValueContext(shape)); + PointBuilder pb = new PointBuilder(-100, 45); + if (shapeMapper instanceof GeoShapeFieldMapper) { + shapeMapper.parse(context.createExternalValueContext(pb.buildLucene())); + } else { + shapeMapper.parse(context.createExternalValueContext(pb.buildS4J())); + } context = context.createExternalValueContext(generatedValue); @@ -210,7 +217,7 @@ public FieldMapper updateFieldType(Map fullNameToFieldT BinaryFieldMapper binMapperUpdate = (BinaryFieldMapper) binMapper.updateFieldType(fullNameToFieldType); BooleanFieldMapper boolMapperUpdate = (BooleanFieldMapper) boolMapper.updateFieldType(fullNameToFieldType); GeoPointFieldMapper pointMapperUpdate = (GeoPointFieldMapper) pointMapper.updateFieldType(fullNameToFieldType); - GeoShapeFieldMapper shapeMapperUpdate = (GeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); + BaseGeoShapeFieldMapper shapeMapperUpdate = (BaseGeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); TextFieldMapper stringMapperUpdate = (TextFieldMapper) stringMapper.updateFieldType(fullNameToFieldType); if (update == this && multiFieldsUpdate == multiFields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java index e1158f77bd47b..6d47e4a784e06 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java @@ -21,12 +21,13 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.builders.PointBuilder; +import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.locationtech.jts.geom.Coordinate; import java.util.Arrays; import java.util.Collection; @@ -118,7 +119,8 @@ public void testExternalValues() throws Exception { assertThat(response.getHits().getTotalHits().value, equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", new PointBuilder(-100, 45)).relation(ShapeRelation.WITHIN)) + .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", + new EnvelopeBuilder(new Coordinate(-101, 46), new Coordinate(-99, 44))).relation(ShapeRelation.WITHIN)) .execute().actionGet(); assertThat(response.getHits().getTotalHits().value, equalTo((long) 1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 20e689e9d7e89..a5e2d7c31afe2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -18,14 +18,9 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -42,7 +37,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.not; public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { @@ -53,10 +47,10 @@ protected Collection> getPlugins() { public void testDefaultConfiguration() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -64,12 +58,8 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.025d)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoShapeFieldMapper.Defaults.GEOHASH_LEVELS)); - assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION)); + assertThat(geoShapeFieldMapper.fieldType().orientation(), + equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); } /** @@ -77,11 +67,11 @@ public void testDefaultConfiguration() throws IOException { */ public void testOrientationParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -95,11 +85,11 @@ public void testOrientationParsing() throws IOException { // explicit right orientation test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -117,11 +107,11 @@ public void testOrientationParsing() throws IOException { */ public void testCoerceParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "true") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "true") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -133,11 +123,11 @@ public void testCoerceParsing() throws IOException { // explicit false coerce test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "false") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "false") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -146,6 +136,7 @@ public void testCoerceParsing() throws IOException { coerce = ((GeoShapeFieldMapper)fieldMapper).coerce().value(); assertThat(coerce, equalTo(false)); + assertFieldWarnings("tree"); } @@ -222,304 +213,45 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } - public void testGeohashConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "4") - .field("distance_error_pct", "0.1") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.1)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); - } - - public void testQuadtreeConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("distance_error_pct", "0.5") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - } - - public void testLevelPrecisionConfiguration() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "26") - .field("precision", "70m") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and - // "tree_levels" setting distErrPct to 0 to guarantee desired precision - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is less precise so it loses - assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); - } - } - - public void testPointsOnlyOption() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - } - - public void testLevelDefaults() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); + private void assertFieldWarnings(String... fieldNames) { + String[] warnings = new String[fieldNames.length]; + for (int i = 0; i < fieldNames.length; ++i) { + warnings[i] = "Field parameter [" + fieldNames[i] + "] " + + "is deprecated and will be removed in a future version."; } } public void testGeoShapeMapperMerge() throws Exception { String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("shape").field("type", "geo_shape").field("tree", "geohash") - .field("strategy", "recursive") - .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) - .field("orientation", "ccw") - .endObject().endObject().endObject().endObject()); + .startObject("shape").field("type", "geo_shape") + .field("orientation", "ccw") + .endObject().endObject().endObject().endObject()); MapperService mapperService = createIndex("test").mapperService(); DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), MapperService.MergeReason.MAPPING_UPDATE); String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("tree", "quadtree") - .field("strategy", "term").field("precision", "1km") - .field("tree_levels", 26).field("distance_error_pct", 26) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); - try { - mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); - fail(); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); - } + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); // verify nothing changed Mapper fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.01)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); - // correct mapping + // change mapping; orientation stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m") - .field("tree_levels", 8).field("distance_error_pct", 0.001) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("orientation", "cw").endObject().endObject().endObject().endObject()); docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.001)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); } @@ -544,112 +276,12 @@ public void testSerializeDefaults() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":21")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":9")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertFalse(serialized, serialized.contains("\"precision\":")); - assertTrue(serialized, serialized.contains("\"tree_levels\":6")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6") .endObject().endObject() .endObject().endObject()); DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertFalse(serialized, serialized.contains("\"tree_levels\":")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() + "\"")); } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6m") - .field("tree_levels", "5") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":5")); - } - } - - public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - // term strategy changes the default for points_only, check that we handle it correctly - assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); - } - - - public void testPointsOnlyFalseWithTermStrategy() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .field("points_only", false) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) - ); - assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); } public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java index a1c225f8a0657..c10ec5facf806 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java @@ -18,69 +18,23 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.index.mapper.GeoShapeFieldMapper.GeoShapeFieldType; import org.junit.Before; -import java.io.IOException; - public class GeoShapeFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { - return new GeoShapeFieldMapper.GeoShapeFieldType(); + return new GeoShapeFieldType(); } @Before public void setupProperties() { - addModifier(new Modifier("tree", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTree("quadtree"); - } - }); - addModifier(new Modifier("strategy", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setStrategyName("term"); - } - }); - addModifier(new Modifier("tree_levels", false) { + addModifier(new FieldTypeTestCase.Modifier("orientation", true) { @Override public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTreeLevels(10); + ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); } }); - addModifier(new Modifier("precision", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setPrecisionInMeters(20); - } - }); - addModifier(new Modifier("distance_error_pct", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); - } - }); - addModifier(new Modifier("orientation", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); - } - }); - } - - /** - * Test for {@link GeoShapeFieldType#setStrategyName(String)} that checks that {@link GeoShapeFieldType#pointsOnly()} - * gets set as a side effect when using SpatialStrategy.TERM - */ - public void testSetStrategyName() throws IOException { - GeoShapeFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategyName(SpatialStrategy.RECURSIVE.getStrategyName()); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategyName(SpatialStrategy.TERM.getStrategyName()); - assertTrue(fieldType.pointsOnly()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java new file mode 100644 index 0000000000000..11d8c72531db2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -0,0 +1,714 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; + +public class LegacyGeoShapeFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + public void testDefaultConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("strategy", "recursive") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + assertThat(geoShapeFieldMapper.fieldType().tree(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.TREE)); + assertThat(geoShapeFieldMapper.fieldType().treeLevels(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.QUADTREE_LEVELS)); + assertThat(geoShapeFieldMapper.fieldType().pointsOnly(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.POINTS_ONLY)); + assertThat(geoShapeFieldMapper.fieldType().distanceErrorPct(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.DISTANCE_ERROR_PCT)); + assertThat(geoShapeFieldMapper.fieldType().orientation(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.ORIENTATION.value())); + assertFieldWarnings("strategy"); + } + + /** + * Test that orientation parameter correctly parses + */ + public void testOrientationParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ShapeBuilder.Orientation orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); + + // explicit right orientation test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); + assertFieldWarnings("tree"); + } + + /** + * Test that coerce parameter correctly parses + */ + public void testCoerceParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("coerce", "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + boolean coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); + assertThat(coerce, equalTo(true)); + + // explicit false coerce test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("coerce", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); + assertThat(coerce, equalTo(false)); + assertFieldWarnings("tree"); + } + + + /** + * Test that accept_z_value parameter correctly parses + */ + public void testIgnoreZValue() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("strategy", "recursive") + .field(IGNORE_Z_VALUE.getPreferredName(), "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + boolean ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); + assertThat(ignoreZValue, equalTo(true)); + + // explicit false accept_z_value test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field(IGNORE_Z_VALUE.getPreferredName(), "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); + assertThat(ignoreZValue, equalTo(false)); + assertFieldWarnings("strategy", "tree"); + } + + /** + * Test that ignore_malformed parameter correctly parses + */ + public void testIgnoreMalformedParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("ignore_malformed", "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + Explicit ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.value(), equalTo(true)); + + // explicit false ignore_malformed test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("ignore_malformed", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.explicit(), equalTo(true)); + assertThat(ignoreMalformed.value(), equalTo(false)); + assertFieldWarnings("tree"); + } + + public void testGeohashConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "4") + .field("distance_error_pct", "0.1") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.1)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct"); + } + + public void testQuadtreeConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("distance_error_pct", "0.5") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct", "points_only"); + } + + private void assertFieldWarnings(String... fieldNames) { + String[] warnings = new String[fieldNames.length]; + for (int i = 0; i < fieldNames.length; ++i) { + warnings[i] = "Field parameter [" + fieldNames[i] + "] " + + "is deprecated and will be removed in a future version."; + } + assertWarnings(warnings); + } + + public void testLevelPrecisionConfiguration() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "26") + .field("precision", "70m") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and + // "tree_levels" setting distErrPct to 0 to guarantee desired precision + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is less precise so it loses + assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); + } + assertFieldWarnings("tree", "tree_levels", "precision", "distance_error_pct"); + } + + public void testPointsOnlyOption() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "points_only"); + } + + public void testLevelDefaults() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); + } + assertFieldWarnings("tree", "distance_error_pct"); + } + + public void testGeoShapeMapperMerge() throws Exception { + String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("shape").field("type", "geo_shape").field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) + .field("orientation", "ccw") + .endObject().endObject().endObject().endObject()); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), + MapperService.MergeReason.MAPPING_UPDATE); + String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "quadtree") + .field("strategy", "term").field("precision", "1km") + .field("tree_levels", 26).field("distance_error_pct", 26) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); + } + + // verify nothing changed + Mapper fieldMapper = docMapper.mappers().getMapper("shape"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.01)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); + assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); + + // correct mapping + stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m") + .field("tree_levels", 8).field("distance_error_pct", 0.001) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + + fieldMapper = docMapper.mappers().getMapper("shape"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.001)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); + assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); + + assertFieldWarnings("tree", "strategy", "precision", "tree_levels", "distance_error_pct"); + } + + public void testEmptyName() throws Exception { + // after 5.x + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> parser.parse("type1", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("name cannot be empty string")); + assertFieldWarnings("tree"); + } + + public void testSerializeDefaults() throws Exception { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":9")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertFalse(serialized, serialized.contains("\"precision\":")); + assertTrue(serialized, serialized.contains("\"tree_levels\":6")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":10")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6m") + .field("tree_levels", "5") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":5")); + } + assertFieldWarnings("tree", "tree_levels", "precision"); + } + + public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + // term strategy changes the default for points_only, check that we handle it correctly + assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); + assertFieldWarnings("tree", "precision", "strategy"); + } + + + public void testPointsOnlyFalseWithTermStrategy() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .field("points_only", false) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> parser.parse("type1", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); + assertFieldWarnings("tree", "precision", "strategy", "points_only"); + } + + public String toXContentString(LegacyGeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + ToXContent.Params params; + if (includeDefaults) { + params = new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true")); + } else { + params = ToXContent.EMPTY_PARAMS; + } + mapper.doXContentBody(builder, includeDefaults, params); + return Strings.toString(builder.endObject()); + } + + public String toXContentString(LegacyGeoShapeFieldMapper mapper) throws IOException { + return toXContentString(mapper, true); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java new file mode 100644 index 0000000000000..2fcbed82e33b4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType; +import org.junit.Before; + +import java.io.IOException; + +public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { + @Override + protected MappedFieldType createDefaultFieldType() { + return new GeoShapeFieldType(); + } + + @Before + public void setupProperties() { + addModifier(new Modifier("tree", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setTree("geohash"); + } + }); + addModifier(new Modifier("strategy", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setStrategy(SpatialStrategy.TERM); + } + }); + addModifier(new Modifier("tree_levels", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setTreeLevels(10); + } + }); + addModifier(new Modifier("precision", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setPrecisionInMeters(20); + } + }); + addModifier(new Modifier("distance_error_pct", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); + } + }); + addModifier(new Modifier("orientation", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); + } + }); + } + + /** + * Test for {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks + * that {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#pointsOnly()} gets set as a side effect when using SpatialStrategy.TERM + */ + public void testSetStrategyName() throws IOException { + GeoShapeFieldType fieldType = new GeoShapeFieldType(); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategy(SpatialStrategy.RECURSIVE); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategy(SpatialStrategy.TERM); + assertTrue(fieldType.pointsOnly()); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index bcd2b4ef14440..e2e4db1f9b790 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanQuery; @@ -29,7 +28,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -54,29 +52,41 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase { - private static String indexedShapeId; - private static String indexedShapeType; - private static String indexedShapePath; - private static String indexedShapeIndex; - private static String indexedShapeRouting; - private static ShapeBuilder indexedShapeToReturn; + protected static String indexedShapeId; + protected static String indexedShapeType; + protected static String indexedShapePath; + protected static String indexedShapeIndex; + protected static String indexedShapeRouting; + protected static ShapeBuilder indexedShapeToReturn; + + @Override + protected boolean enableWarningsCheck() { + return false; + } + + protected String fieldName() { + return GEO_SHAPE_FIELD_NAME; + } @Override protected GeoShapeQueryBuilder doCreateTestQueryBuilder() { return doCreateTestQueryBuilder(randomBoolean()); } - private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - ShapeType shapeType = ShapeType.randomType(random()); + + protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + // LatLonShape does not support MultiPoint queries + RandomShapeGenerator.ShapeType shapeType = + randomFrom(ShapeType.POINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON); ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { - builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder = new GeoShapeQueryBuilder(fieldName(), shape); } else { indexedShapeToReturn = shape; indexedShapeId = randomAlphaOfLengthBetween(3, 20); indexedShapeType = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeId, indexedShapeType); + builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); if (randomBoolean()) { indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); builder.indexedShapeIndex(indexedShapeIndex); @@ -91,15 +101,11 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { } } if (randomBoolean()) { - SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); - // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so - // we try to avoid that combination - while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { - strategy = randomFrom(SpatialStrategy.values()); - } - builder.strategy(strategy); - if (strategy != SpatialStrategy.TERM) { - builder.relation(randomFrom(ShapeRelation.values())); + if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { + builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS)); + } else { + // LatLonShape does not support CONTAINS: + builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN)); } } @@ -161,41 +167,28 @@ public void testNoFieldName() throws Exception { } public void testNoShape() throws IOException { - expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null)); + expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(fieldName(), null)); } public void testNoIndexedShape() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null, "type")); + () -> new GeoShapeQueryBuilder(fieldName(), null, "type")); assertEquals("either shapeBytes or indexedShapeId and indexedShapeType are required", e.getMessage()); } public void testNoIndexedShapeType() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, "id", null)); + () -> new GeoShapeQueryBuilder(fieldName(), "id", null)); assertEquals("indexedShapeType is required if indexedShapeId is specified", e.getMessage()); } public void testNoRelation() throws IOException { ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(fieldName(), shape); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.relation(null)); assertEquals("No Shape Relation defined", e.getMessage()); } - public void testInvalidRelation() throws IOException { - ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); - expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); - GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder3.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - } - // see #3878 public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(0, 0), new Coordinate(10, 10)); @@ -205,7 +198,7 @@ public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { public void testFromJson() throws IOException { String json = - "{\n" + + "{\n" + " \"geo_shape\" : {\n" + " \"location\" : {\n" + " \"shape\" : {\n" + @@ -230,7 +223,7 @@ public void testMustRewrite() throws IOException { UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> query.toQuery(createShardContext())); assertEquals("query must be rewritten first", e.getMessage()); QueryBuilder rewrite = rewriteAndFetch(query, createShardContext()); - GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); + GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); geoShapeQueryBuilder.strategy(query.strategy()); geoShapeQueryBuilder.relation(query.relation()); assertEquals(geoShapeQueryBuilder, rewrite); @@ -244,7 +237,7 @@ public void testMultipleRewrite() throws IOException { builder = rewriteAndFetch(builder, createShardContext()); - GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); + GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); expectedShape.strategy(shape.strategy()); expectedShape.relation(shape.relation()); QueryBuilder expected = new BoolQueryBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java new file mode 100644 index 0000000000000..3cf6f2031810a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.test.geo.RandomShapeGenerator; +import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; + +import java.io.IOException; + +public class LegacyGeoShapeFieldQueryTests extends GeoShapeQueryBuilderTests { + + @Override + protected String fieldName() { + return LEGACY_GEO_SHAPE_FIELD_NAME; + } + + @Override + protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + ShapeType shapeType = ShapeType.randomType(random()); + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); + GeoShapeQueryBuilder builder; + clearShapeFields(); + if (indexedShape == false) { + builder = new GeoShapeQueryBuilder(fieldName(), shape); + } else { + indexedShapeToReturn = shape; + indexedShapeId = randomAlphaOfLengthBetween(3, 20); + indexedShapeType = randomAlphaOfLengthBetween(3, 20); + builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); + if (randomBoolean()) { + indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeIndex(indexedShapeIndex); + } + if (randomBoolean()) { + indexedShapePath = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapePath(indexedShapePath); + } + if (randomBoolean()) { + indexedShapeRouting = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeRouting(indexedShapeRouting); + } + } + if (randomBoolean()) { + SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); + // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so + // we try to avoid that combination + while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { + strategy = randomFrom(SpatialStrategy.values()); + } + builder.strategy(strategy); + if (strategy != SpatialStrategy.TERM) { + builder.relation(randomFrom(ShapeRelation.values())); + } + } + + if (randomBoolean()) { + builder.ignoreUnmapped(randomBoolean()); + } + return builder; + } + + public void testInvalidRelation() throws IOException { + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); + expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); + GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder3.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index 1067ed62db46e..184ee2759c15e 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -62,6 +62,7 @@ import static org.hamcrest.Matchers.notNullValue; public class MatchQueryBuilderTests extends AbstractQueryTestCase { + @Override protected MatchQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME, diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 70f504516ec8a..1c34057457a4c 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1048,6 +1048,12 @@ public void testDisabledFieldNamesField() throws Exception { "_field_names", "enabled=true"))), MapperService.MergeReason.MAPPING_UPDATE); } + assertWarnings(new String[] { + "Field parameter [tree_levels] is deprecated and will be removed in a future version.", + "Field parameter [precision] is deprecated and will be removed in a future version.", + "Field parameter [strategy] is deprecated and will be removed in a future version.", + "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." + }); } diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index 7231dc7f9a9f2..b26a7ff510a3d 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -380,6 +380,7 @@ public void testBulk() throws Exception { .endObject() .startObject("location") .field("type", "geo_shape") + .field("ignore_malformed", true) .endObject() .endObject() .endObject() diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index 854872730e8fd..b120b54687607 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -45,21 +45,21 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { public void testOrientationPersistence() throws Exception { String idxName = "orientation"; String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); // create index assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); ensureGreen(idxName, idxName+"2"); @@ -144,9 +144,8 @@ public void testIndexShapeRouting() throws Exception { String source = "{\n" + " \"shape\" : {\n" + - " \"type\" : \"circle\",\n" + - " \"coordinates\" : [-45.0, 45.0],\n" + - " \"radius\" : \"100m\"\n" + + " \"type\" : \"bbox\",\n" + + " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 6c90dcf59e952..a64f98df5a6eb 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -19,16 +19,21 @@ package org.elasticsearch.search.geo; +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import org.apache.lucene.geo.GeoTestUtil; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.LineStringBuilder; +import org.elasticsearch.common.geo.builders.MultiPointBuilder; +import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.settings.Settings; @@ -36,7 +41,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -63,12 +70,26 @@ import static org.hamcrest.Matchers.nullValue; public class GeoShapeQueryTests extends ESSingleNodeTestCase { + private static final String[] PREFIX_TREES = new String[] { + LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.GEOHASH, + LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.QUADTREE + }; + + private XContentBuilder createMapping() throws Exception { + XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape"); + if (randomBoolean()) { + xcb = xcb.field("tree", randomFrom(PREFIX_TREES)) + .field("strategy", randomFrom(SpatialStrategy.RECURSIVE, SpatialStrategy.TERM)); + } + xcb = xcb.endObject().endObject().endObject().endObject(); + + return xcb; + } + public void testNullShape() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -79,12 +100,7 @@ public void testNullShape() throws Exception { } public void testIndexPointsFilterRectangle() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -126,12 +142,11 @@ public void testIndexPointsFilterRectangle() throws Exception { } public void testEdgeCases() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject().endObject().endObject(); + String mapping = Strings.toString(xcb); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -163,12 +178,7 @@ public void testEdgeCases() throws Exception { } public void testIndexedShapeReference() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); createIndex("shapes"); ensureGreen(); @@ -205,14 +215,7 @@ public void testIndexedShapeReference() throws Exception { } public void testIndexedShapeReferenceSourceDisabled() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject() - .endObject() - .endObject(); + XContentBuilder mapping = createMapping(); client().admin().indices().prepareCreate("test").addMapping("type1", mapping).get(); createIndex("shapes", Settings.EMPTY, "shape_type", "_source", "enabled=false"); ensureGreen(); @@ -326,24 +329,107 @@ public void testShapeFetchingPath() throws Exception { assertHitCount(result, 1); } - public void testShapeFilterWithRandomGeoCollection() throws Exception { + public void testQueryRandomGeoCollection() throws Exception { // Create a random geometry collection. GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); + CoordinatesBuilder cb = new CoordinatesBuilder(); + for (int i = 0; i < randomPoly.numPoints(); ++i) { + cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); + } + gcb.shape(new PolygonBuilder(cb)); logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") - .get(); + if (randomBoolean()) { + client().admin().indices().prepareCreate("test") + .addMapping("type", "location", "type=geo_shape").get(); + } else { + client().admin().indices().prepareCreate("test") + .addMapping("type", "location", "type=geo_shape,tree=quadtree").get(); + } XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); - ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1))); + ShapeBuilder filterShape = (gcb.getShapeAt(gcb.numShapes() - 1)); - GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape); - filter.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) - .setPostFilter(filter).get(); + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", filterShape); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); + assertSearchResponse(result); + assertHitCount(result, 1); + } + + public void testRandomGeoCollectionQuery() throws Exception { + boolean usePrefixTrees = randomBoolean(); + // Create a random geometry collection to index. + GeometryCollectionBuilder gcb; + if (usePrefixTrees) { + gcb = RandomShapeGenerator.createGeometryCollection(random()); + } else { + // vector strategy does not yet support multipoint queries + gcb = new GeometryCollectionBuilder(); + int numShapes = RandomNumbers.randomIntBetween(random(), 1, 4); + for (int i = 0; i < numShapes; ++i) { + ShapeBuilder shape; + do { + shape = RandomShapeGenerator.createShape(random()); + } while (shape instanceof MultiPointBuilder); + gcb.shape(shape); + } + } + org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); + CoordinatesBuilder cb = new CoordinatesBuilder(); + for (int i = 0; i < randomPoly.numPoints(); ++i) { + cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); + } + gcb.shape(new PolygonBuilder(cb)); + + logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); + + if (usePrefixTrees == false) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + // Create a random geometry collection to query + GeometryCollectionBuilder queryCollection = RandomShapeGenerator.createGeometryCollection(random()); + queryCollection.shape(new PolygonBuilder(cb)); + + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", queryCollection); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); + assertSearchResponse(result); + assertTrue(result.getHits().getTotalHits().value > 0); + } + + /** tests querying a random geometry collection with a point */ + public void testPointQuery() throws Exception { + // Create a random geometry collection to index. + GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + double[] pt = new double[] {GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()}; + PointBuilder pb = new PointBuilder(pt[0], pt[1]); + gcb.shape(pb); + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", pb); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); assertSearchResponse(result); assertHitCount(result, 1); } @@ -375,6 +461,28 @@ public void testContainsShapeQuery() throws Exception { assertThat(response.getHits().getTotalHits().value, greaterThan(0L)); } + public void testExistsQuery() throws Exception { + // Create a random geometry collection. + GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); + + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + ExistsQueryBuilder eqb = QueryBuilders.existsQuery("location"); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(eqb).get(); + assertSearchResponse(result); + assertHitCount(result, 1); + } + public void testShapeFilterWithDefinedGeoCollection() throws Exception { createIndex("shapes"); client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") diff --git a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java new file mode 100644 index 0000000000000..574bdd46bba5b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.geo; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class LegacyGeoShapeIntegrationIT extends ESIntegTestCase { + + /** + * Test that orientation parameter correctly persists across cluster restart + */ + public void testOrientationPersistence() throws Exception { + String idxName = "orientation"; + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); + + // create index + assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); + + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); + + assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); + ensureGreen(idxName, idxName+"2"); + + internalCluster().fullRestart(); + ensureGreen(idxName, idxName+"2"); + + // left orientation test + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName)); + IndexService indexService = indicesService.indexService(resolveIndex(idxName)); + MappedFieldType fieldType = indexService.mapperService().fullName("location"); + assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); + + LegacyGeoShapeFieldMapper.GeoShapeFieldType gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; + ShapeBuilder.Orientation orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); + + // right orientation test + indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2")); + indexService = indicesService.indexService(resolveIndex((idxName+"2"))); + fieldType = indexService.mapperService().fullName("location"); + assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); + + gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; + orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); + } + + /** + * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document + */ + public void testIgnoreMalformed() throws Exception { + // create index + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("geometry", "shape", "type=geo_shape,tree=quadtree,ignore_malformed=true").get()); + ensureGreen(); + + // test self crossing ccw poly not crossing dateline + String polygonGeoJson = Strings.toString(XContentFactory.jsonBuilder().startObject().field("type", "Polygon") + .startArray("coordinates") + .startArray() + .startArray().value(176.0).value(15.0).endArray() + .startArray().value(-177.0).value(10.0).endArray() + .startArray().value(-177.0).value(-10.0).endArray() + .startArray().value(176.0).value(-15.0).endArray() + .startArray().value(-177.0).value(15.0).endArray() + .startArray().value(172.0).value(0.0).endArray() + .startArray().value(176.0).value(15.0).endArray() + .endArray() + .endArray() + .endObject()); + + indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape", + polygonGeoJson)); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); + } + + /** + * Test that the indexed shape routing can be provided if it is required + */ + public void testIndexShapeRouting() throws Exception { + String mapping = "{\n" + + " \"_routing\": {\n" + + " \"required\": true\n" + + " },\n" + + " \"properties\": {\n" + + " \"shape\": {\n" + + " \"type\": \"geo_shape\",\n" + + " \"tree\" : \"quadtree\"\n" + + " }\n" + + " }\n" + + " }"; + + + // create index + assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping, XContentType.JSON).get()); + ensureGreen(); + + String source = "{\n" + + " \"shape\" : {\n" + + " \"type\" : \"bbox\",\n" + + " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + + " }\n" + + "}"; + + indexRandom(true, client().prepareIndex("test", "doc", "0").setSource(source, XContentType.JSON).setRouting("ABC")); + + SearchResponse searchResponse = client().prepareSearch("test").setQuery( + geoShapeQuery("shape", "0", "doc").indexedShapeIndex("test").indexedShapeRouting("ABC") + ).get(); + + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); + } + + private String findNodeName(String index) { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); + String nodeId = shard.assignedShards().get(0).currentNodeId(); + return state.getNodes().get(nodeId).getName(); + } +} diff --git a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 76d18a59f9f63..0d964e8eb6fa7 100644 --- a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.search.geo.GeoShapeQueryTests; import org.junit.Assert; import org.locationtech.spatial4j.context.jts.JtsSpatialContext; import org.locationtech.spatial4j.distance.DistanceUtils; @@ -153,6 +154,7 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit /** * Creates a random shape useful for randomized testing, NOTE: exercise caution when using this to build random GeometryCollections * as creating a large random number of random shapes can result in massive resource consumption + * see: {@link GeoShapeQueryTests#testQueryRandomGeoCollection()} * * The following options are included * @param nearPoint Create a shape near a provided point diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 5eef0a249b687..daf29e46b0519 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -113,6 +113,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String LEGACY_GEO_SHAPE_FIELD_NAME = "mapped_legacy_geo_shape"; protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, @@ -217,12 +218,28 @@ public void beforeTest() throws Exception { AbstractBuilderTestCase.this, false); return null; }); + if (enableWarningsCheck() == true) { + assertDeprecatedGeoWarnings(); + } } serviceHolder.clientInvocationHandler.delegate = this; serviceHolderWithNoType.clientInvocationHandler.delegate = this; } + protected void assertDeprecatedGeoWarnings() { + String prefix = "Field parameter ["; + String postfix = "] is deprecated and will be removed in a future version."; + String[] deprecationWarnings = new String[] { + prefix + "tree" + postfix, + prefix + "tree_levels" + postfix, + prefix + "precision" + postfix, + prefix + "strategy" + postfix, + prefix + "distance_error_pct" + postfix + }; + assertWarnings(deprecationWarnings); + } + protected static SearchContext getSearchContext(QueryShardContext context) { TestSearchContext testSearchContext = new TestSearchContext(context) { @Override @@ -396,7 +413,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { OBJECT_FIELD_NAME, "type=object", GEO_POINT_FIELD_NAME, "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, "type=geo_shape" + GEO_SHAPE_FIELD_NAME, "type=geo_shape", + LEGACY_GEO_SHAPE_FIELD_NAME, "type=geo_shape,tree=quadtree" ))), MapperService.MergeReason.MAPPING_UPDATE); // also add mappings for two inner field in the object field mapperService.merge("_doc", new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\"," From 36ef19107cc5021ff171ee3a8eff4a9336935f4f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 17 Dec 2018 22:34:27 +0100 Subject: [PATCH 46/57] TESTS:Debug Log. IndexStatsIT#testFilterCacheStats --- .../test/java/org/elasticsearch/indices/stats/IndexStatsIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 152429ae4357b..5fb67a64d9db5 100644 --- a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -62,6 +62,7 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.io.IOException; import java.util.ArrayList; @@ -1007,6 +1008,7 @@ private void assertCumulativeQueryCacheStats(IndicesStatsResponse response) { assertEquals(total, shardTotal); } + @TestLogging("_root:DEBUG") // this fails at a very low rate on CI: https://github.com/elastic/elasticsearch/issues/32506 public void testFilterCacheStats() throws Exception { Settings settings = Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build(); assertAcked(prepareCreate("index").setSettings(settings).get()); From 47c923df917af98a552360120bd9ab5536305354 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 17 Dec 2018 16:25:11 -0600 Subject: [PATCH 47/57] ingest: support default pipelines + bulk upserts (#36618) This commit adds support to enable bulk upserts to use an index's default pipeline. Bulk upsert, doc_as_upsert, and script_as_upsert are all supported. However, bulk script_as_upsert has slightly surprising behavior since the pipeline is executed _before_ the script is evaluated. This means that the pipeline only has access the data found in the upsert field of the script_as_upsert. The non-bulk script_as_upsert (existing behavior) runs the pipeline _after_ the script is executed. This commit does _not_ attempt to consolidate the bulk and non-bulk behavior for script_as_upsert. This commit also adds additional testing for the non-bulk behavior, which remains unchanged with this commit. fixes #36219 --- .../test/ingest/200_default_pipeline.yml | 101 ++++++++++++++++-- .../action/bulk/TransportBulkAction.java | 30 ++++-- .../elasticsearch/ingest/IngestService.java | 10 +- .../bulk/TransportBulkActionIngestTests.java | 54 ++++++++++ .../action/bulk/TransportBulkActionTests.java | 21 ++++ 5 files changed, 196 insertions(+), 20 deletions(-) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml index 4695991f3c3b1..d4b39c5e99ac2 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml @@ -23,7 +23,7 @@ teardown: ] } - match: { acknowledged: true } - +# default pipeline via index - do: indices.create: index: test @@ -48,7 +48,7 @@ teardown: id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } - +# default pipeline via alias - do: index: index: test_alias @@ -63,12 +63,101 @@ teardown: id: 2 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } +# default pipeline via upsert + - do: + update: + index: test + type: test + id: 3 + body: + script: + source: "ctx._source.ran_script = true" + lang: "painless" + upsert: { "bytes_source_field":"1kb" } + - do: + get: + index: test + type: test + id: 3 + - match: { _source.bytes_source_field: "1kb" } + - match: { _source.bytes_target_field: 1024 } +# default pipeline via scripted upsert + - do: + update: + index: test + type: test + id: 4 + body: + script: + source: "ctx._source.bytes_source_field = '1kb'" + lang: "painless" + upsert : {} + scripted_upsert: true + - do: + get: + index: test + type: test + id: 4 + - match: { _source.bytes_source_field: "1kb" } + - match: { _source.bytes_target_field: 1024 } +# default pipeline via doc_as_upsert + - do: + update: + index: test + type: test + id: 5 + body: + doc: { "bytes_source_field":"1kb" } + doc_as_upsert: true + - do: + get: + index: test + type: test + id: 5 + - match: { _source.bytes_source_field: "1kb" } + - match: { _source.bytes_target_field: 1024 } +# default pipeline via bulk upsert +# note - bulk scripted upsert's execute the pipeline before the script, so any data referenced by the pipeline +# needs to be in the upsert, not the script + - do: + bulk: + refresh: true + body: | + {"update":{"_id":"6","_index":"test","_type":"test"}} + {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} + {"update":{"_id":"7","_index":"test","_type":"test"}} + {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} + {"update":{"_id":"8","_index":"test","_type":"test"}} + {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} + - do: + mget: + body: + docs: + - { _index: "test", _type: "_doc", _id: "6" } + - { _index: "test", _type: "_doc", _id: "7" } + - { _index: "test", _type: "_doc", _id: "8" } + - match: { docs.0._index: "test" } + - match: { docs.0._id: "6" } + - match: { docs.0._source.bytes_source_field: "1kb" } + - match: { docs.0._source.bytes_target_field: 1024 } + - is_false: docs.0._source.ran_script + - match: { docs.1._index: "test" } + - match: { docs.1._id: "7" } + - match: { docs.1._source.bytes_source_field: "2kb" } + - match: { docs.1._source.bytes_target_field: 2048 } + - match: { docs.2._index: "test" } + - match: { docs.2._id: "8" } + - match: { docs.2._source.bytes_source_field: "3kb" } + - match: { docs.2._source.bytes_target_field: 3072 } + - match: { docs.2._source.ran_script: true } + +# explicit no default pipeline - do: index: index: test type: test - id: 3 + id: 9 pipeline: "_none" body: {bytes_source_field: "1kb"} @@ -76,15 +165,15 @@ teardown: get: index: test type: test - id: 3 + id: 9 - match: { _source.bytes_source_field: "1kb" } - is_false: _source.bytes_target_field - +# bad request - do: catch: bad_request index: index: test type: test - id: 4 + id: 10 pipeline: "" body: {bytes_source_field: "1kb"} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index fa294a1bb2b6c..a89d162979f5f 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -127,6 +127,24 @@ public TransportBulkAction(ThreadPool threadPool, TransportService transportServ clusterService.addStateApplier(this.ingestForwarder); } + /** + * Retrieves the {@link IndexRequest} from the provided {@link DocWriteRequest} for index or upsert actions. Upserts are + * modeled as {@link IndexRequest} inside the {@link UpdateRequest}. Ignores {@link org.elasticsearch.action.delete.DeleteRequest}'s + * + * @param docWriteRequest The request to find the {@link IndexRequest} + * @return the found {@link IndexRequest} or {@code null} if one can not be found. + */ + public static IndexRequest getIndexWriteRequest(DocWriteRequest docWriteRequest) { + IndexRequest indexRequest = null; + if (docWriteRequest instanceof IndexRequest) { + indexRequest = (IndexRequest) docWriteRequest; + } else if (docWriteRequest instanceof UpdateRequest) { + UpdateRequest updateRequest = (UpdateRequest) docWriteRequest; + indexRequest = updateRequest.docAsUpsert() ? updateRequest.doc() : updateRequest.upsertRequest(); + } + return indexRequest; + } + @Override protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener listener) { final long startTime = relativeTime(); @@ -207,12 +225,12 @@ private void executeIngestAndBulk(Task task, final BulkRequest bulkRequest, fina final MetaData metaData = clusterService.state().getMetaData(); ImmutableOpenMap indicesMetaData = metaData.indices(); for (DocWriteRequest actionRequest : bulkRequest.requests) { - if (actionRequest instanceof IndexRequest) { - IndexRequest indexRequest = (IndexRequest) actionRequest; + IndexRequest indexRequest = getIndexWriteRequest(actionRequest); + if(indexRequest != null){ String pipeline = indexRequest.getPipeline(); if (pipeline == null) { - IndexMetaData indexMetaData = indicesMetaData.get(indexRequest.index()); - if (indexMetaData == null) { + IndexMetaData indexMetaData = indicesMetaData.get(actionRequest.index()); + if (indexMetaData == null && indexRequest.index() != null) { //check the alias AliasOrIndex indexOrAlias = metaData.getAliasAndIndexLookup().get(indexRequest.index()); if (indexOrAlias != null && indexOrAlias.isAlias()) { @@ -626,7 +644,7 @@ ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, } void markCurrentItemAsDropped() { - IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot); + IndexRequest indexRequest = getIndexWriteRequest(bulkRequest.requests().get(currentSlot)); failedSlots.set(currentSlot); itemResponses.add( new BulkItemResponse(currentSlot, indexRequest.opType(), @@ -639,7 +657,7 @@ void markCurrentItemAsDropped() { } void markCurrentItemAsFailed(Exception e) { - IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot); + IndexRequest indexRequest = getIndexWriteRequest(bulkRequest.requests().get(currentSlot)); // We hit a error during preprocessing a request, so we: // 1) Remember the request item slot from the bulk, so that we're done processing all requests we know what failed // 2) Add a bulk item failure for this request diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 705e77028a1ef..6951e33d5e741 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -24,11 +24,11 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.TransportBulkAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -388,13 +388,7 @@ public void onFailure(Exception e) { @Override protected void doRun() { for (DocWriteRequest actionRequest : actionRequests) { - IndexRequest indexRequest = null; - if (actionRequest instanceof IndexRequest) { - indexRequest = (IndexRequest) actionRequest; - } else if (actionRequest instanceof UpdateRequest) { - UpdateRequest updateRequest = (UpdateRequest) actionRequest; - indexRequest = updateRequest.docAsUpsert() ? updateRequest.doc() : updateRequest.upsertRequest(); - } + IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest); if (indexRequest == null) { continue; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index f25f8844153a5..219aee9ebe2ff 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; @@ -408,6 +409,57 @@ public void testUseDefaultPipelineWithAlias() throws Exception { validateDefaultPipeline(new IndexRequest(WITH_DEFAULT_PIPELINE_ALIAS, "type", "id")); } + public void testUseDefaultPipelineWithBulkUpsert() throws Exception { + Exception exception = new Exception("fake exception"); + BulkRequest bulkRequest = new BulkRequest(); + IndexRequest indexRequest1 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id1").source(Collections.emptyMap()); + IndexRequest indexRequest2 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id2").source(Collections.emptyMap()); + IndexRequest indexRequest3 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id3").source(Collections.emptyMap()); + UpdateRequest upsertRequest = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id1").upsert(indexRequest1).script(mockScript("1")); + UpdateRequest docAsUpsertRequest = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id2").doc(indexRequest2).docAsUpsert(true); + // this test only covers the mechanics that scripted bulk upserts will execute a default pipeline. However, in practice scripted + // bulk upserts with a default pipeline are a bit surprising since the script executes AFTER the pipeline. + UpdateRequest scriptedUpsert = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id2").upsert(indexRequest3).script(mockScript("1")) + .scriptedUpsert(true); + bulkRequest.add(upsertRequest).add(docAsUpsertRequest).add(scriptedUpsert); + + AtomicBoolean responseCalled = new AtomicBoolean(false); + AtomicBoolean failureCalled = new AtomicBoolean(false); + assertNull(indexRequest1.getPipeline()); + assertNull(indexRequest2.getPipeline()); + assertNull(indexRequest3.getPipeline()); + action.execute(null, bulkRequest, ActionListener.wrap( + response -> { + BulkItemResponse itemResponse = response.iterator().next(); + assertThat(itemResponse.getFailure().getMessage(), containsString("fake exception")); + responseCalled.set(true); + }, + e -> { + assertThat(e, sameInstance(exception)); + failureCalled.set(true); + })); + + // check failure works, and passes through to the listener + assertFalse(action.isExecuted); // haven't executed yet + assertFalse(responseCalled.get()); + assertFalse(failureCalled.get()); + verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); + assertEquals(indexRequest1.getPipeline(), "default_pipeline"); + assertEquals(indexRequest2.getPipeline(), "default_pipeline"); + assertEquals(indexRequest3.getPipeline(), "default_pipeline"); + completionHandler.getValue().accept(exception); + assertTrue(failureCalled.get()); + + // now check success of the transport bulk action + indexRequest1.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing + indexRequest2.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing + indexRequest3.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing + completionHandler.getValue().accept(null); + assertTrue(action.isExecuted); + assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one + verifyZeroInteractions(transportService); + } + public void testCreateIndexBeforeRunPipeline() throws Exception { Exception exception = new Exception("fake exception"); IndexRequest indexRequest = new IndexRequest("missing_index", "type", "id"); @@ -445,6 +497,7 @@ private void validateDefaultPipeline(IndexRequest indexRequest) { indexRequest.source(Collections.emptyMap()); AtomicBoolean responseCalled = new AtomicBoolean(false); AtomicBoolean failureCalled = new AtomicBoolean(false); + assertNull(indexRequest.getPipeline()); singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap( response -> { responseCalled.set(true); @@ -459,6 +512,7 @@ private void validateDefaultPipeline(IndexRequest indexRequest) { assertFalse(responseCalled.get()); assertFalse(failureCalled.get()); verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); + assertEquals(indexRequest.getPipeline(), "default_pipeline"); completionHandler.getValue().accept(exception); assertTrue(failureCalled.get()); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index a058cf477411f..162ef56553df4 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -23,8 +23,10 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.TransportBulkActionTookTests.Resolver; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -132,4 +134,23 @@ public void testDeleteNonExistingDocExternalGteVersionCreatesIndex() throws Exce throw new AssertionError(exception); })); } + + public void testGetIndexWriteRequest() throws Exception { + IndexRequest indexRequest = new IndexRequest("index", "type", "id1").source(Collections.emptyMap()); + UpdateRequest upsertRequest = new UpdateRequest("index", "type", "id1").upsert(indexRequest).script(mockScript("1")); + UpdateRequest docAsUpsertRequest = new UpdateRequest("index", "type", "id2").doc(indexRequest).docAsUpsert(true); + UpdateRequest scriptedUpsert = new UpdateRequest("index", "type", "id2").upsert(indexRequest).script(mockScript("1")) + .scriptedUpsert(true); + + assertEquals(TransportBulkAction.getIndexWriteRequest(indexRequest), indexRequest); + assertEquals(TransportBulkAction.getIndexWriteRequest(upsertRequest), indexRequest); + assertEquals(TransportBulkAction.getIndexWriteRequest(docAsUpsertRequest), indexRequest); + assertEquals(TransportBulkAction.getIndexWriteRequest(scriptedUpsert), indexRequest); + + DeleteRequest deleteRequest = new DeleteRequest("index", "id"); + assertNull(TransportBulkAction.getIndexWriteRequest(deleteRequest)); + + UpdateRequest badUpsertRequest = new UpdateRequest("index", "type", "id1"); + assertNull(TransportBulkAction.getIndexWriteRequest(badUpsertRequest)); + } } From 2ae0fa2b29243d285f481b97129aefba19f10fa0 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 17 Dec 2018 17:42:58 -0500 Subject: [PATCH 48/57] Fix duplicate phrase in shrink/split error message (#36734) This commit removes a duplicate "must be a" from the shrink/split error messages. --- .../org/elasticsearch/cluster/metadata/IndexMetaData.java | 4 ++-- .../elasticsearch/cluster/metadata/IndexMetaDataTests.java | 4 ++-- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index f1dd843d7987f..5d23971dddbe4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1531,14 +1531,14 @@ public static int getRoutingFactor(int sourceNumberOfShards, int targetNumberOfS if (sourceNumberOfShards < targetNumberOfShards) { // split factor = targetNumberOfShards / sourceNumberOfShards; if (factor * sourceNumberOfShards != targetNumberOfShards || factor <= 1) { - throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a " + "factor of [" + targetNumberOfShards + "]"); } } else if (sourceNumberOfShards > targetNumberOfShards) { // shrink factor = sourceNumberOfShards / targetNumberOfShards; if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { - throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a " + "multiple of [" + targetNumberOfShards + "]"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 393f7f6b1d4aa..1fdea596afbf9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -227,7 +227,7 @@ public void testSelectSplitShard() { assertEquals("the number of target shards (0) must be greater than the shard id: 0", expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 0)).getMessage()); - assertEquals("the number of source shards [2] must be a must be a factor of [3]", + assertEquals("the number of source shards [2] must be a factor of [3]", expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 3)).getMessage()); assertEquals("the number of routing shards [4] must be a multiple of the target shards [8]", @@ -285,6 +285,6 @@ public void testNumberOfRoutingShards() { Settings notAFactorySettings = Settings.builder().put("index.number_of_shards", 2).put("index.number_of_routing_shards", 3).build(); iae = expectThrows(IllegalArgumentException.class, () -> IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(notAFactorySettings)); - assertEquals("the number of source shards [2] must be a must be a factor of [3]", iae.getMessage()); + assertEquals("the number of source shards [2] must be a factor of [3]", iae.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 6cbd83e5b242c..ec89e085f0784 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -154,7 +154,7 @@ public void testValidateShrinkIndex() { MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); - assertEquals("the number of source shards [8] must be a must be a multiple of [3]", + assertEquals("the number of source shards [8] must be a multiple of [3]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 8, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", @@ -221,7 +221,7 @@ public void testValidateSplitIndex() { ).getMessage()); - assertEquals("the number of source shards [3] must be a must be a factor of [4]", + assertEquals("the number of source shards [3] must be a factor of [4]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", 3, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", From 0aebd20c6edf0ecee97637bc3fa23316b805bbad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 18 Dec 2018 00:57:42 +0100 Subject: [PATCH 49/57] Deprecate types in get_source and exist_source (#36426) This change adds a new untyped endpoint `{index}/_source/{id}` for both the GET and the HEAD methods to get the source of a document or check for its existance. It also adds deprecation warnings to RestGetSourceAction that emit a warning when the old deprecated "type" parameter is still used. Also updating documentation and tests where appropriate. Relates to #35190 --- .../client/RequestConverters.java | 10 +++- .../client/RequestConvertersTests.java | 54 +++++++++++++++++++ .../documentation/CRUDDocumentationIT.java | 1 - docs/reference/docs/get.asciidoc | 14 ++--- .../rest/Netty4HeadBodyIsEmptyIT.java | 6 +-- .../rest-api-spec/api/exists_source.json | 8 +-- .../rest-api-spec/api/get_source.json | 8 +-- .../test/get_source/10_basic.yml | 8 +-- .../test/get_source/11_basic_with_types.yml | 17 ++++++ .../test/get_source/15_default_values.yml | 7 ++- .../16_default_values_with_types.yml | 16 ++++++ .../test/get_source/40_routing.yml | 7 ++- .../test/get_source/41_routing_with_types.yml | 42 +++++++++++++++ .../test/get_source/60_realtime_refresh.yml | 6 +-- .../61_realtime_refresh_with_types.yml | 49 +++++++++++++++++ .../test/get_source/70_source_filtering.yml | 11 ++-- .../71_source_filtering_with_types.yml | 27 ++++++++++ .../test/get_source/80_missing.yml | 12 ++++- .../test/get_source/81_missing_with_types.yml | 19 +++++++ .../test/get_source/85_source_missing.yml | 7 ++- .../86_source_missing_with_types.yml | 38 +++++++++++++ .../elasticsearch/action/get/GetRequest.java | 4 +- .../action/document/RestGetSourceAction.java | 16 +++++- .../action/get/GetRequestTests.java | 1 + .../document/RestGetSourceActionTests.java | 49 +++++++++++++++-- 25 files changed, 393 insertions(+), 44 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/11_basic_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/16_default_values_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/41_routing_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/61_realtime_refresh_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/71_source_filtering_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/81_missing_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/86_source_missing_with_types.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 6791b5f825966..c7a54a9ac32fe 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -268,8 +268,14 @@ private static Request getStyleRequest(String method, GetRequest getRequest) { } static Request sourceExists(GetRequest getRequest) { - Request request = new Request(HttpHead.METHOD_NAME, endpoint(getRequest.index(), getRequest.type(), getRequest.id(), "_source")); - + String optionalType = getRequest.type(); + String endpoint; + if (optionalType.equals(MapperService.SINGLE_MAPPING_NAME)) { + endpoint = endpoint(getRequest.index(), "_source", getRequest.id()); + } else { + endpoint = endpoint(getRequest.index(), optionalType, getRequest.id(), "_source"); + } + Request request = new Request(HttpHead.METHOD_NAME, endpoint); Params parameters = new Params(request); parameters.withPreference(getRequest.preference()); parameters.withRouting(getRequest.routing()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index b50d2c1265ed2..fa0f1c5708c2b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -73,6 +73,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.rankeval.PrecisionAtK; @@ -115,6 +116,7 @@ import java.util.Locale; import java.util.Map; import java.util.StringJoiner; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -156,6 +158,58 @@ public void testGetWithType() { getAndExistsWithTypeTest(RequestConverters::get, HttpGet.METHOD_NAME); } + public void testSourceExists() throws IOException { + doTestSourceExists((index, id) -> new GetRequest(index, id)); + } + + public void testSourceExistsWithType() throws IOException { + String type = frequently() ? randomAlphaOfLengthBetween(3, 10) : MapperService.SINGLE_MAPPING_NAME; + doTestSourceExists((index, id) -> new GetRequest(index, type, id)); + } + + private static void doTestSourceExists(BiFunction requestFunction) throws IOException { + String index = randomAlphaOfLengthBetween(3, 10); + String id = randomAlphaOfLengthBetween(3, 10); + final GetRequest getRequest = requestFunction.apply(index, id); + + Map expectedParams = new HashMap<>(); + if (randomBoolean()) { + String preference = randomAlphaOfLengthBetween(3, 10); + getRequest.preference(preference); + expectedParams.put("preference", preference); + } + if (randomBoolean()) { + String routing = randomAlphaOfLengthBetween(3, 10); + getRequest.routing(routing); + expectedParams.put("routing", routing); + } + if (randomBoolean()) { + boolean realtime = randomBoolean(); + getRequest.realtime(realtime); + if (realtime == false) { + expectedParams.put("realtime", "false"); + } + } + if (randomBoolean()) { + boolean refresh = randomBoolean(); + getRequest.refresh(refresh); + if (refresh) { + expectedParams.put("refresh", "true"); + } + } + Request request = RequestConverters.sourceExists(getRequest); + assertEquals(HttpHead.METHOD_NAME, request.getMethod()); + String type = getRequest.type(); + if (type.equals(MapperService.SINGLE_MAPPING_NAME)) { + assertEquals("/" + index + "/_source/" + id, request.getEndpoint()); + } else { + assertEquals("/" + index + "/" + type + "/" + id + "/_source", request.getEndpoint()); + } + + assertEquals(expectedParams, request.getParameters()); + assertNull(request.getEntity()); + } + public void testMultiGet() throws IOException { Map expectedParams = new HashMap<>(); MultiGetRequest multiGetRequest = new MultiGetRequest(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index a9430b67aef0f..5279c19a415c4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -1265,7 +1265,6 @@ public void testGet() throws Exception { assertEquals(3, getResponse.getSourceAsMap().size()); //tag::get-response String index = getResponse.getIndex(); - String type = getResponse.getType(); String id = getResponse.getId(); if (getResponse.isExists()) { long version = getResponse.getVersion(); diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc index 3ab5fa11fa179..5271b976f9677 100644 --- a/docs/reference/docs/get.asciidoc +++ b/docs/reference/docs/get.asciidoc @@ -1,9 +1,9 @@ [[docs-get]] == Get API -The get API allows to get a typed JSON document from the index based on +The get API allows to get a JSON document from the index based on its id. The following example gets a JSON document from an index called -twitter, under a type called `_doc`, with id valued 0: +twitter with id valued 0: [source,js] -------------------------------------------------- @@ -34,7 +34,7 @@ The result of the above get operation is: -------------------------------------------------- // TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] -The above result includes the `_index`, `_type`, `_id` and `_version` +The above result includes the `_index`, `_id` and `_version` of the document we wish to retrieve, including the actual `_source` of the document if it could be found (as indicated by the `found` field in the response). @@ -223,13 +223,13 @@ will fail. [[_source]] === Getting the +_source+ directly -Use the `/{index}/{type}/{id}/_source` endpoint to get +Use the `/{index}/_source/{id}` endpoint to get just the `_source` field of the document, without any additional content around it. For example: [source,js] -------------------------------------------------- -GET twitter/_doc/1/_source +GET twitter/_source/1 -------------------------------------------------- // CONSOLE // TEST[continued] @@ -238,7 +238,7 @@ You can also use the same source filtering parameters to control which parts of [source,js] -------------------------------------------------- -GET twitter/_doc/1/_source?_source_includes=*.id&_source_excludes=entities' +GET twitter/_source/1/?_source_includes=*.id&_source_excludes=entities' -------------------------------------------------- // CONSOLE // TEST[continued] @@ -248,7 +248,7 @@ An existing document will not have a _source if it is disabled in the < * The operation requires the {@link #index()}, {@link #type(String)} and {@link #id(String)} @@ -84,7 +84,6 @@ public GetRequest(String index) { * @param index The index to get the document from * @param type The type of the document * @param id The id of the document - * * @deprecated Types are in the process of being removed, use {@link GetRequest(String, String)} instead. */ @Deprecated @@ -127,7 +126,6 @@ public ActionRequestValidationException validate() { /** * Sets the type of the document to fetch. - * * @deprecated Types are in the process of being removed. */ @Deprecated diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index c48529d420c1e..af376bf7c3ccf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -19,12 +19,14 @@ package org.elasticsearch.rest.action.document; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -49,8 +51,14 @@ */ public class RestGetSourceAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetSourceAction.class)); + static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get_source and exist_source" + + "requests is deprecated."; + public RestGetSourceAction(final Settings settings, final RestController controller) { super(settings); + controller.registerHandler(GET, "/{index}/_source/{id}", this); + controller.registerHandler(HEAD, "/{index}/_source/{id}", this); controller.registerHandler(GET, "/{index}/{type}/{id}/_source", this); controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this); } @@ -62,7 +70,13 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id")); + final GetRequest getRequest; + if (request.hasParam("type")) { + deprecationLogger.deprecatedAndMaybeLog("get_source_with_types", TYPES_DEPRECATION_MESSAGE); + getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id")); + } else { + getRequest = new GetRequest(request.param("index"), request.param("id")); + } getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh())); getRequest.routing(request.param("routing")); getRequest.preference(request.param("preference")); diff --git a/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java index fc2162e866240..499932ccdf026 100644 --- a/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java @@ -40,6 +40,7 @@ public void testValidation() { final ActionRequestValidationException validate = request.validate(); assertThat(validate, not(nullValue())); + assertEquals(2, validate.validationErrors().size()); assertThat(validate.validationErrors(), hasItems("type is missing", "id is missing")); } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java index e8f573dc57ddc..f012c1393c9ad 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java @@ -23,26 +23,38 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestRequest.Method; import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.rest.action.document.RestGetSourceAction.RestGetSourceResponseListener; import org.elasticsearch.test.rest.FakeRestChannel; import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; import org.junit.AfterClass; +import org.junit.Before; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.rest.RestStatus.OK; -import static org.elasticsearch.rest.action.document.RestGetSourceAction.RestGetSourceResponseListener; import static org.hamcrest.Matchers.equalTo; -public class RestGetSourceActionTests extends ESTestCase { +public class RestGetSourceActionTests extends RestActionTestCase { private static RestRequest request = new FakeRestRequest(); private static FakeRestChannel channel = new FakeRestChannel(request, true, 0); private static RestGetSourceResponseListener listener = new RestGetSourceResponseListener(channel, request); + @Before + public void setUpAction() { + new RestGetSourceAction(Settings.EMPTY, controller()); + } + @AfterClass public static void cleanupReferences() { request = null; @@ -50,6 +62,37 @@ public static void cleanupReferences() { listener = null; } + /** + * test deprecation is logged if type is used in path + */ + public void testTypeInPath() { + for (Method method : Arrays.asList(Method.GET, Method.HEAD)) { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(method) + .withPath("/some_index/some_type/id/_source") + .build(); + dispatchRequest(request); + assertWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE); + } + } + + /** + * test deprecation is logged if type is used as parameter + */ + public void testTypeParameter() { + Map params = new HashMap<>(); + params.put("type", "some_type"); + for (Method method : Arrays.asList(Method.GET, Method.HEAD)) { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(method) + .withPath("/some_index/_source/id") + .withParams(params) + .build(); + dispatchRequest(request); + assertWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE); + } + } + public void testRestGetSourceAction() throws Exception { final BytesReference source = new BytesArray("{\"foo\": \"bar\"}"); final GetResponse response = From b8f9238be37d0f206415fca3d67f2d6235c4b078 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 17 Dec 2018 20:09:46 -0600 Subject: [PATCH 50/57] Revert "[Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320)" This reverts commit 5bc7822562a6eefa4a64743233160cdc9f431adf. --- .../mapping/types/geo-shape.asciidoc | 186 ++--- .../migration/migrate_7_0/mappings.asciidoc | 16 - .../query-dsl/geo-shape-query.asciidoc | 5 +- .../common/geo/ShapeRelation.java | 12 - .../builders/GeometryCollectionBuilder.java | 3 + .../common/geo/parsers/GeoJsonParser.java | 24 +- .../common/geo/parsers/GeoWKTParser.java | 13 +- .../common/geo/parsers/ShapeParser.java | 4 +- .../index/mapper/BaseGeoShapeFieldMapper.java | 336 --------- .../index/mapper/GeoShapeFieldMapper.java | 610 +++++++++++++-- .../mapper/LegacyGeoShapeFieldMapper.java | 596 --------------- .../index/query/GeoShapeQueryBuilder.java | 117 +-- .../elasticsearch/indices/IndicesModule.java | 8 +- .../common/geo/GeoJsonShapeParserTests.java | 8 +- .../common/geo/GeoWKTShapeParserTests.java | 19 +- .../index/mapper/ExternalMapper.java | 21 +- .../ExternalValuesMapperIntegrationIT.java | 6 +- .../mapper/GeoShapeFieldMapperTests.java | 452 +++++++++-- .../index/mapper/GeoShapeFieldTypeTests.java | 52 +- .../LegacyGeoShapeFieldMapperTests.java | 714 ------------------ .../mapper/LegacyGeoShapeFieldTypeTests.java | 86 --- .../query/GeoShapeQueryBuilderTests.java | 75 +- .../query/LegacyGeoShapeFieldQueryTests.java | 94 --- .../index/query/MatchQueryBuilderTests.java | 1 - .../query/QueryStringQueryBuilderTests.java | 6 - .../elasticsearch/search/geo/GeoFilterIT.java | 1 - .../search/geo/GeoShapeIntegrationIT.java | 25 +- .../search/geo/GeoShapeQueryTests.java | 186 +---- .../geo/LegacyGeoShapeIntegrationIT.java | 170 ----- .../test/geo/RandomShapeGenerator.java | 2 - .../test/AbstractBuilderTestCase.java | 20 +- 31 files changed, 1233 insertions(+), 2635 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java delete mode 100644 server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java delete mode 100644 server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 8efb184afa6ba..2f51465d1109f 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -21,59 +21,48 @@ type. |======================================================================= |Option |Description| Default -|`tree |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree -implementation to be used: `geohash` for GeohashPrefixTree and `quadtree` -for QuadPrefixTree. Note: This parameter is only relevant for `term` and -`recursive` strategies. -| `quadtree` - -|`precision` |deprecated[6.6, PrefixTrees no longer used] This parameter may -be used instead of `tree_levels` to set an appropriate value for the -`tree_levels` parameter. The value specifies the desired precision and -Elasticsearch will calculate the best tree_levels value to honor this -precision. The value should be a number followed by an optional distance -unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, -`miles`, `km`, `kilometers`, `m`,`meters`, `cm`,`centimeters`, `mm`, -`millimeters`. Note: This parameter is only relevant for `term` and -`recursive` strategies. +|`tree` |Name of the PrefixTree implementation to be used: `geohash` for +GeohashPrefixTree and `quadtree` for QuadPrefixTree. +| `geohash` + +|`precision` |This parameter may be used instead of `tree_levels` to set +an appropriate value for the `tree_levels` parameter. The value +specifies the desired precision and Elasticsearch will calculate the +best tree_levels value to honor this precision. The value should be a +number followed by an optional distance unit. Valid distance units +include: `in`, `inch`, `yd`, `yard`, `mi`, `miles`, `km`, `kilometers`, +`m`,`meters`, `cm`,`centimeters`, `mm`, `millimeters`. | `50m` -|`tree_levels` |deprecated[6.6, PrefixTrees no longer used] Maximum number -of layers to be used by the PrefixTree. This can be used to control the -precision of shape representations andtherefore how many terms are -indexed. Defaults to the default value of the chosen PrefixTree -implementation. Since this parameter requires a certain level of -understanding of the underlying implementation, users may use the -`precision` parameter instead. However, Elasticsearch only uses the -tree_levels parameter internally and this is what is returned via the -mapping API even if you use the precision parameter. Note: This parameter -is only relevant for `term` and `recursive` strategies. +|`tree_levels` |Maximum number of layers to be used by the PrefixTree. +This can be used to control the precision of shape representations and +therefore how many terms are indexed. Defaults to the default value of +the chosen PrefixTree implementation. Since this parameter requires a +certain level of understanding of the underlying implementation, users +may use the `precision` parameter instead. However, Elasticsearch only +uses the tree_levels parameter internally and this is what is returned +via the mapping API even if you use the precision parameter. | various -|`strategy` |deprecated[6.6, PrefixTrees no longer used] The strategy -parameter defines the approach for how to represent shapes at indexing -and search time. It also influences the capabilities available so it -is recommended to let Elasticsearch set this parameter automatically. -There are two strategies available: `recursive`, and `term`. -Recursive and Term strategies are deprecated and will be removed in a -future version. While they are still available, the Term strategy -supports point types only (the `points_only` parameter will be -automatically set to true) while Recursive strategy supports all -shape types. (IMPORTANT: see <> for more -detailed information about these strategies) +|`strategy` |The strategy parameter defines the approach for how to +represent shapes at indexing and search time. It also influences the +capabilities available so it is recommended to let Elasticsearch set +this parameter automatically. There are two strategies available: +`recursive` and `term`. Term strategy supports point types only (the +`points_only` parameter will be automatically set to true) while +Recursive strategy supports all shape types. (IMPORTANT: see +<> for more detailed information) | `recursive` -|`distance_error_pct` |deprecated[6.6, PrefixTrees no longer used] Used as a -hint to the PrefixTree about how precise it should be. Defaults to 0.025 (2.5%) -with 0.5 as the maximum supported value. PERFORMANCE NOTE: This value will -default to 0 if a `precision` or `tree_level` definition is explicitly defined. -This guarantees spatial precision at the level defined in the mapping. This can -lead to significant memory usage for high resolution shapes with low error -(e.g., large shapes at 1m with < 0.001 error). To improve indexing performance -(at the cost of query accuracy) explicitly define `tree_level` or `precision` -along with a reasonable `distance_error_pct`, noting that large shapes will have -greater false positives. Note: This parameter is only relevant for `term` and -`recursive` strategies. +|`distance_error_pct` |Used as a hint to the PrefixTree about how +precise it should be. Defaults to 0.025 (2.5%) with 0.5 as the maximum +supported value. PERFORMANCE NOTE: This value will default to 0 if a `precision` or +`tree_level` definition is explicitly defined. This guarantees spatial precision +at the level defined in the mapping. This can lead to significant memory usage +for high resolution shapes with low error (e.g., large shapes at 1m with < 0.001 error). +To improve indexing performance (at the cost of query accuracy) explicitly define +`tree_level` or `precision` along with a reasonable `distance_error_pct`, noting +that large shapes will have greater false positives. | `0.025` |`orientation` |Optionally define how to interpret vertex order for @@ -88,13 +77,13 @@ sets vertex order for the coordinate list of a geo_shape field but can be overridden in each individual GeoJSON or WKT document. | `ccw` -|`points_only` |deprecated[6.6, PrefixTrees no longer used] Setting this option to -`true` (defaults to `false`) configures the `geo_shape` field type for point -shapes only (NOTE: Multi-Points are not yet supported). This optimizes index and -search performance for the `geohash` and `quadtree` when it is known that only points -will be indexed. At present geo_shape queries can not be executed on `geo_point` -field types. This option bridges the gap by improving point performance on a -`geo_shape` field so that `geo_shape` queries are optimal on a point only field. +|`points_only` |Setting this option to `true` (defaults to `false`) configures +the `geo_shape` field type for point shapes only (NOTE: Multi-Points are not +yet supported). This optimizes index and search performance for the `geohash` and +`quadtree` when it is known that only points will be indexed. At present geo_shape +queries can not be executed on `geo_point` field types. This option bridges the gap +by improving point performance on a `geo_shape` field so that `geo_shape` queries are +optimal on a point only field. | `false` |`ignore_malformed` |If true, malformed GeoJSON or WKT shapes are ignored. If @@ -111,35 +100,16 @@ and reject the whole document. |======================================================================= - -[[geoshape-indexing-approach]] -[float] -==== Indexing approach -GeoShape types are indexed by decomposing the shape into a triangular mesh and -indexing each triangle as a 7 dimension point in a BKD tree. This provides -near perfect spatial resolution (down to 1e-7 decimal degree precision) since all -spatial relations are computed using an encoded vector representation of the -original shape instead of a raster-grid representation as used by the -<> indexing approach. Performance of the tessellator primarily -depends on the number of vertices that define the polygon/multi-polyogn. While -this is the default indexing technique prefix trees can still be used by setting -the `tree` or `strategy` parameters according to the appropriate -<>. Note that these parameters are now deprecated -and will be removed in a future version. - [[prefix-trees]] [float] ==== Prefix trees -deprecated[6.6, PrefixTrees no longer used] To efficiently represent shapes in -an inverted index, Shapes are converted into a series of hashes representing -grid squares (commonly referred to as "rasters") using implementations of a -PrefixTree. The tree notion comes from the fact that the PrefixTree uses multiple -grid layers, each with an increasing level of precision to represent the Earth. -This can be thought of as increasing the level of detail of a map or image at higher -zoom levels. Since this approach causes precision issues with indexed shape, it has -been deprecated in favor of a vector indexing approach that indexes the shapes as a -triangular mesh (see <>). +To efficiently represent shapes in the index, Shapes are converted into +a series of hashes representing grid squares (commonly referred to as "rasters") +using implementations of a PrefixTree. The tree notion comes from the fact that +the PrefixTree uses multiple grid layers, each with an increasing level of +precision to represent the Earth. This can be thought of as increasing the level +of detail of a map or image at higher zoom levels. Multiple PrefixTree implementations are provided: @@ -161,10 +131,9 @@ number of levels for the quad trees in Elasticsearch is 29; the default is 21. [[spatial-strategy]] [float] ===== Spatial strategies -deprecated[6.6, PrefixTrees no longer used] The indexing implementation -selected relies on a SpatialStrategy for choosing how to decompose the shapes -(either as grid squares or a tessellated triangular mesh). Each strategy -answers the following: +The PrefixTree implementations rely on a SpatialStrategy for decomposing +the provided Shape(s) into approximated grid squares. Each strategy answers +the following: * What type of Shapes can be indexed? * What types of Query Operations and Shapes can be used? @@ -177,7 +146,7 @@ are provided: |======================================================================= |Strategy |Supported Shapes |Supported Queries |Multiple Shapes -|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes +|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes |`term` |<> |`INTERSECTS` |Yes |======================================================================= @@ -185,13 +154,13 @@ are provided: [float] ===== Accuracy -`Recursive` and `Term` strategies do not provide 100% accuracy and depending on -how they are configured it may return some false positives for `INTERSECTS`, -`WITHIN` and `CONTAINS` queries, and some false negatives for `DISJOINT` queries. -To mitigate this, it is important to select an appropriate value for the tree_levels -parameter and to adjust expectations accordingly. For example, a point may be near -the border of a particular grid cell and may thus not match a query that only matches -the cell right next to it -- even though the shape is very close to the point. +Geo_shape does not provide 100% accuracy and depending on how it is configured +it may return some false positives for `INTERSECTS`, `WITHIN` and `CONTAINS` +queries, and some false negatives for `DISJOINT` queries. To mitigate this, it +is important to select an appropriate value for the tree_levels parameter and +to adjust expectations accordingly. For example, a point may be near the border +of a particular grid cell and may thus not match a query that only matches the +cell right next to it -- even though the shape is very close to the point. [float] ===== Example @@ -204,7 +173,9 @@ PUT /example "doc": { "properties": { "location": { - "type": "geo_shape" + "type": "geo_shape", + "tree": "quadtree", + "precision": "100m" } } } @@ -214,23 +185,22 @@ PUT /example // CONSOLE // TESTSETUP -This mapping definition maps the location field to the geo_shape -type using the default vector implementation. It provides -approximately 1e-7 decimal degree precision. +This mapping maps the location field to the geo_shape type using the +quad_tree implementation and a precision of 100m. Elasticsearch translates +this into a tree_levels setting of 20. [float] -===== Performance considerations with Prefix Trees +===== Performance considerations -deprecated[6.6, PrefixTrees no longer used] With prefix trees, -Elasticsearch uses the paths in the tree as terms in the inverted index -and in queries. The higher the level (and thus the precision), the more -terms are generated. Of course, calculating the terms, keeping them in +Elasticsearch uses the paths in the prefix tree as terms in the index +and in queries. The higher the level is (and thus the precision), the +more terms are generated. Of course, calculating the terms, keeping them in memory, and storing them on disk all have a price. Especially with higher -tree levels, indices can become extremely large even with a modest amount -of data. Additionally, the size of the features also matters. Big, complex -polygons can take up a lot of space at higher tree levels. Which setting -is right depends on the use case. Generally one trades off accuracy against -index size and query performance. +tree levels, indices can become extremely large even with a modest +amount of data. Additionally, the size of the features also matters. +Big, complex polygons can take up a lot of space at higher tree levels. +Which setting is right depends on the use case. Generally one trades off +accuracy against index size and query performance. The defaults in Elasticsearch for both implementations are a compromise between index size and a reasonable level of precision of 50m at the @@ -628,10 +598,7 @@ POST /example/doc ===== Circle Elasticsearch supports a `circle` type, which consists of a center -point with a radius. Note that this circle representation can only -be indexed when using the `recursive` Prefix Tree strategy. For -the default <> circles should be approximated using -a `POLYGON`. +point with a radius: [source,js] -------------------------------------------------- @@ -645,7 +612,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[skip:not supported in default] Note: The inner `radius` field is required. If not specified, then the units of the `radius` will default to `METERS`. diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index f08ea3ab89c1d..5ee1615796c98 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -52,19 +52,3 @@ as a better alternative. An error will now be thrown when unknown configuration options are provided to similarities. Such unknown parameters were ignored before. - -[float] -==== deprecated `geo_shape` Prefix Tree indexing - -`geo_shape` types now default to using a vector indexing approach based on Lucene's new -`LatLonShape` field type. This indexes shapes as a triangular mesh instead of decomposing -them into individual grid cells. To index using legacy prefix trees `recursive` or `term` -strategy must be explicitly defined. Note that these strategies are now deprecated and will -be removed in a future version. - -[float] -==== deprecated `geo_shape` parameters - -The following type parameters are deprecated for the `geo_shape` field type: `tree`, -`precision`, `tree_levels`, `distance_error_pct`, `points_only`, and `strategy`. They -will be removed in a future version. \ No newline at end of file diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc index f796881d520c6..4e00a2f49b475 100644 --- a/docs/reference/query-dsl/geo-shape-query.asciidoc +++ b/docs/reference/query-dsl/geo-shape-query.asciidoc @@ -7,7 +7,7 @@ Requires the <>. The `geo_shape` query uses the same grid square representation as the `geo_shape` mapping to find documents that have a shape that intersects -with the query shape. It will also use the same Prefix Tree configuration +with the query shape. It will also use the same PrefixTree configuration as defined for the field mapping. The query supports two ways of defining the query shape, either by @@ -157,8 +157,7 @@ has nothing in common with the query geometry. * `WITHIN` - Return all documents whose `geo_shape` field is within the query geometry. * `CONTAINS` - Return all documents whose `geo_shape` field -contains the query geometry. Note: this is only supported using the -`recursive` Prefix Tree Strategy deprecated[6.6] +contains the query geometry. [float] ==== Ignore Unmapped diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java index e2e177c8f0fd2..e83e18ce43255 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.geo; -import org.apache.lucene.document.LatLonShape.QueryRelation; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -63,17 +62,6 @@ public static ShapeRelation getRelationByName(String name) { return null; } - /** Maps ShapeRelation to Lucene's LatLonShapeRelation */ - public QueryRelation getLuceneRelation() { - switch (this) { - case INTERSECTS: return QueryRelation.INTERSECTS; - case DISJOINT: return QueryRelation.DISJOINT; - case WITHIN: return QueryRelation.WITHIN; - default: - throw new IllegalArgumentException("ShapeRelation [" + this + "] not supported"); - } - } - public String getRelationName() { return relationName; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index fdf7073bd7454..b6e94c012c603 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -197,6 +197,9 @@ public Object buildLucene() { } } + if (shapes.size() == 1) { + return shapes.get(0); + } return shapes.toArray(new Object[shapes.size()]); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java index b008786ed9211..4f0586711e439 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java @@ -25,11 +25,10 @@ import org.elasticsearch.common.geo.builders.CircleBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentSubParser; -import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -42,22 +41,17 @@ * complies with geojson specification: https://tools.ietf.org/html/rfc7946 */ abstract class GeoJsonParser { - protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) + protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException { GeoShapeType shapeType = null; DistanceUnit.Distance radius = null; CoordinateNode coordinateNode = null; GeometryCollectionBuilder geometryCollections = null; - Orientation orientation = (shapeMapper == null) - ? BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() - : shapeMapper.orientation(); - Explicit coerce = (shapeMapper == null) - ? BaseGeoShapeFieldMapper.Defaults.COERCE - : shapeMapper.coerce(); - Explicit ignoreZValue = (shapeMapper == null) - ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE - : shapeMapper.ignoreZValue(); + ShapeBuilder.Orientation requestedOrientation = + (shapeMapper == null) ? ShapeBuilder.Orientation.RIGHT : shapeMapper.fieldType().orientation(); + Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); + Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); String malformedException = null; @@ -108,7 +102,7 @@ protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapp malformedException = "cannot have [" + ShapeParser.FIELD_ORIENTATION + "] with type set to [" + shapeType + "]"; } subParser.nextToken(); - orientation = ShapeBuilder.Orientation.fromString(subParser.text()); + requestedOrientation = ShapeBuilder.Orientation.fromString(subParser.text()); } else { subParser.nextToken(); subParser.skipChildren(); @@ -134,7 +128,7 @@ protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapp return geometryCollections; } - return shapeType.getBuilder(coordinateNode, radius, orientation, coerce.value()); + return shapeType.getBuilder(coordinateNode, radius, requestedOrientation, coerce.value()); } /** @@ -208,7 +202,7 @@ private static Coordinate parseCoordinate(XContentParser parser, boolean ignoreZ * @return Geometry[] geometries of the GeometryCollection * @throws IOException Thrown if an error occurs while reading from the XContentParser */ - static GeometryCollectionBuilder parseGeometries(XContentParser parser, BaseGeoShapeFieldMapper mapper) throws + static GeometryCollectionBuilder parseGeometries(XContentParser parser, GeoShapeFieldMapper mapper) throws IOException { if (parser.currentToken() != XContentParser.Token.START_ARRAY) { throw new ElasticsearchParseException("geometries must be an array of geojson objects"); diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java index bf26980c92651..e1d990f0cff25 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java @@ -34,7 +34,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -63,7 +63,7 @@ public class GeoWKTParser { // no instance private GeoWKTParser() {} - public static ShapeBuilder parse(XContentParser parser, final BaseGeoShapeFieldMapper shapeMapper) + public static ShapeBuilder parse(XContentParser parser, final GeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { return parseExpectedType(parser, null, shapeMapper); } @@ -75,12 +75,12 @@ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoSha /** throws an exception if the parsed geometry type does not match the expected shape type */ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoShapeType shapeType, - final BaseGeoShapeFieldMapper shapeMapper) + final GeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { try (StringReader reader = new StringReader(parser.text())) { - Explicit ignoreZValue = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : + Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); - Explicit coerce = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); + Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); // setup the tokenizer; configured to read words w/o numbers StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.resetSyntax(); @@ -257,8 +257,7 @@ private static PolygonBuilder parsePolygon(StreamTokenizer stream, final boolean if (nextEmptyOrOpen(stream).equals(EMPTY)) { return null; } - PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), - BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value()); + PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), ShapeBuilder.Orientation.RIGHT); while (nextCloserOrComma(stream).equals(COMMA)) { builder.hole(parseLinearRing(stream, ignoreZValue, coerce)); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java index 21d1bd9f25564..79582c3365bdb 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import java.io.IOException; @@ -46,7 +46,7 @@ public interface ShapeParser { * if the parsers current token has been null * @throws IOException if the input could not be read */ - static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) throws IOException { + static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException { if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return null; } if (parser.currentToken() == XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java deleted file mode 100644 index 3f1e49e525e81..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.elasticsearch.Version; -import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.DeprecatedParameters; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; - -/** - * Base class for {@link GeoShapeFieldMapper} and {@link LegacyGeoShapeFieldMapper} - */ -public abstract class BaseGeoShapeFieldMapper extends FieldMapper { - public static final String CONTENT_TYPE = "geo_shape"; - - public static class Names { - public static final ParseField ORIENTATION = new ParseField("orientation"); - public static final ParseField COERCE = new ParseField("coerce"); - } - - public static class Defaults { - public static final Explicit ORIENTATION = new Explicit<>(Orientation.RIGHT, false); - public static final Explicit COERCE = new Explicit<>(false, false); - public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); - public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); - } - - public abstract static class Builder - extends FieldMapper.Builder { - protected Boolean coerce; - protected Boolean ignoreMalformed; - protected Boolean ignoreZValue; - protected Orientation orientation; - - /** default builder - used for external mapper*/ - public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType) { - super(name, fieldType, defaultFieldType); - } - - public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType, - boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) { - super(name, fieldType, defaultFieldType); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.orientation = orientation; - this.ignoreZValue = ignoreZ; - } - - public Builder coerce(boolean coerce) { - this.coerce = coerce; - return this; - } - - protected Explicit coerce(BuilderContext context) { - if (coerce != null) { - return new Explicit<>(coerce, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); - } - return Defaults.COERCE; - } - - public Builder orientation(Orientation orientation) { - this.orientation = orientation; - return this; - } - - protected Explicit orientation() { - if (orientation != null) { - return new Explicit<>(orientation, true); - } - return Defaults.ORIENTATION; - } - - @Override - protected boolean defaultDocValues(Version indexCreated) { - return false; - } - - public Builder ignoreMalformed(boolean ignoreMalformed) { - this.ignoreMalformed = ignoreMalformed; - return this; - } - - protected Explicit ignoreMalformed(BuilderContext context) { - if (ignoreMalformed != null) { - return new Explicit<>(ignoreMalformed, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); - } - return Defaults.IGNORE_MALFORMED; - } - - protected Explicit ignoreZValue() { - if (ignoreZValue != null) { - return new Explicit<>(ignoreZValue, true); - } - return Defaults.IGNORE_Z_VALUE; - } - - public Builder ignoreZValue(final boolean ignoreZValue) { - this.ignoreZValue = ignoreZValue; - return this; - } - - @Override - protected void setupFieldType(BuilderContext context) { - super.setupFieldType(context); - - // field mapper handles this at build time - // but prefix tree strategies require a name, so throw a similar exception - if (name().isEmpty()) { - throw new IllegalArgumentException("name cannot be empty string"); - } - - BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); - ft.setOrientation(orientation().value()); - } - } - - public static class TypeParser implements Mapper.TypeParser { - - @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - boolean coerce = Defaults.COERCE.value(); - boolean ignoreZ = Defaults.IGNORE_Z_VALUE.value(); - boolean ignoreMalformed = Defaults.IGNORE_MALFORMED.value(); - Orientation orientation = Defaults.ORIENTATION.value(); - DeprecatedParameters deprecatedParameters = new DeprecatedParameters(); - boolean parsedDeprecatedParams = false; - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (DeprecatedParameters.parse(name, fieldName, fieldNode, deprecatedParameters)) { - parsedDeprecatedParams = true; - iterator.remove(); - } else if (Names.ORIENTATION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - orientation = ShapeBuilder.Orientation.fromString(fieldNode.toString()); - iterator.remove(); - } else if (IGNORE_MALFORMED.equals(fieldName)) { - ignoreMalformed = XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed"); - iterator.remove(); - } else if (Names.COERCE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - coerce = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE.getPreferredName()); - iterator.remove(); - } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { - ignoreZ = XContentMapValues.nodeBooleanValue(fieldNode, - name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()); - iterator.remove(); - } - } - return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); - } - - private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, - boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - if (deprecatedParameters != null) { - return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); - } - return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); - } - - private Builder getLegacyBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, - boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - return new LegacyGeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); - } - } - - public abstract static class BaseGeoShapeFieldType extends MappedFieldType { - protected Orientation orientation = Defaults.ORIENTATION.value(); - - protected BaseGeoShapeFieldType() { - setIndexOptions(IndexOptions.DOCS); - setTokenized(false); - setStored(false); - setStoreTermVectors(false); - setOmitNorms(true); - } - - protected BaseGeoShapeFieldType(BaseGeoShapeFieldType ref) { - super(ref); - this.orientation = ref.orientation; - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - BaseGeoShapeFieldType that = (BaseGeoShapeFieldType) o; - return orientation == that.orientation; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), orientation); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts) { - super.checkCompatibility(fieldType, conflicts); - } - - public Orientation orientation() { return this.orientation; } - - public void setOrientation(Orientation orientation) { - checkIfFrozen(); - this.orientation = orientation; - } - - @Override - public Query existsQuery(QueryShardContext context) { - return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); - } - } - - protected Explicit coerce; - protected Explicit ignoreMalformed; - protected Explicit ignoreZValue; - - protected BaseGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, - MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.ignoreZValue = ignoreZValue; - } - - @Override - protected void doMerge(Mapper mergeWith) { - super.doMerge(mergeWith); - BaseGeoShapeFieldMapper gsfm = (BaseGeoShapeFieldMapper)mergeWith; - if (gsfm.coerce.explicit()) { - this.coerce = gsfm.coerce; - } - if (gsfm.ignoreMalformed.explicit()) { - this.ignoreMalformed = gsfm.ignoreMalformed; - } - if (gsfm.ignoreZValue.explicit()) { - this.ignoreZValue = gsfm.ignoreZValue; - } - } - - @Override - protected void parseCreateField(ParseContext context, List fields) throws IOException { - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - builder.field("type", contentType()); - BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); - if (includeDefaults || ft.orientation() != Defaults.ORIENTATION.value()) { - builder.field(Names.ORIENTATION.getPreferredName(), ft.orientation()); - } - if (includeDefaults || coerce.explicit()) { - builder.field(Names.COERCE.getPreferredName(), coerce.value()); - } - if (includeDefaults || ignoreMalformed.explicit()) { - builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); - } - if (includeDefaults || ignoreZValue.explicit()) { - builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); - } - } - - public Explicit coerce() { - return coerce; - } - - public Explicit ignoreMalformed() { - return ignoreMalformed; - } - - public Explicit ignoreZValue() { - return ignoreZValue; - } - - public Orientation orientation() { - return ((BaseGeoShapeFieldType)fieldType).orientation(); - } - - @Override - protected String contentType() { - return CONTENT_TYPE; - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 65ee2e428faa3..7de40fe337d9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,24 +18,48 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LatLonShape; -import org.apache.lucene.geo.Line; -import org.apache.lucene.geo.Polygon; -import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.XShapeCollection; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.jts.JtsGeometry; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; /** - * FieldMapper for indexing {@link org.apache.lucene.document.LatLonShape}s. + * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. *

    * Currently Shapes can only be indexed and can only be queried using * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently @@ -49,128 +73,554 @@ * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] * ] * } - *

    - * or: - *

    - * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) */ -public class GeoShapeFieldMapper extends BaseGeoShapeFieldMapper { +public class GeoShapeFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "geo_shape"; + + public static class Names { + public static final String TREE = "tree"; + public static final String TREE_GEOHASH = "geohash"; + public static final String TREE_QUADTREE = "quadtree"; + public static final String TREE_LEVELS = "tree_levels"; + public static final String TREE_PRESISION = "precision"; + public static final String DISTANCE_ERROR_PCT = "distance_error_pct"; + public static final String ORIENTATION = "orientation"; + public static final String STRATEGY = "strategy"; + public static final String STRATEGY_POINTS_ONLY = "points_only"; + public static final String COERCE = "coerce"; + } + + public static class Defaults { + public static final String TREE = Names.TREE_GEOHASH; + public static final String STRATEGY = SpatialStrategy.RECURSIVE.getStrategyName(); + public static final boolean POINTS_ONLY = false; + public static final int GEOHASH_LEVELS = GeoUtils.geoHashLevelsForPrecision("50m"); + public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision("50m"); + public static final Orientation ORIENTATION = Orientation.RIGHT; + public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d; + public static final Explicit COERCE = new Explicit<>(false, false); + public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); + public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + + public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType(); + + static { + // setting name here is a hack so freeze can be called...instead all these options should be + // moved to the default ctor for GeoShapeFieldType, and defaultFieldType() should be removed from mappers... + FIELD_TYPE.setName("DoesNotExist"); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setStoreTermVectors(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends FieldMapper.Builder { + + private Boolean coerce; + private Boolean ignoreMalformed; + private Boolean ignoreZValue; - public static class Builder extends BaseGeoShapeFieldMapper.Builder { public Builder(String name) { - super (name, new GeoShapeFieldType(), new GeoShapeFieldType()); + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + } + + @Override + public GeoShapeFieldType fieldType() { + return (GeoShapeFieldType)fieldType; } - public Builder(String name, boolean coerce, boolean ignoreMalformed, ShapeBuilder.Orientation orientation, - boolean ignoreZ) { - super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); + public Builder coerce(boolean coerce) { + this.coerce = coerce; + return this; + } + + @Override + protected boolean defaultDocValues(Version indexCreated) { + return false; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit<>(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); + } + return Defaults.COERCE; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return this; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return Defaults.IGNORE_MALFORMED; + } + + protected Explicit ignoreZValue(BuilderContext context) { + if (ignoreZValue != null) { + return new Explicit<>(ignoreZValue, true); + } + return Defaults.IGNORE_Z_VALUE; + } + + public Builder ignoreZValue(final boolean ignoreZValue) { + this.ignoreZValue = ignoreZValue; + return this; } @Override public GeoShapeFieldMapper build(BuilderContext context) { + GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType; + + if (geoShapeFieldType.treeLevels() == 0 && geoShapeFieldType.precisionInMeters() < 0) { + geoShapeFieldType.setDefaultDistanceErrorPct(Defaults.LEGACY_DISTANCE_ERROR_PCT); + } setupFieldType(context); - return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + + return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), ignoreZValue(context), + context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } - public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { - public GeoShapeFieldType() { - super(); + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + Boolean pointsOnly = null; + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (Names.TREE.equals(fieldName)) { + builder.fieldType().setTree(fieldNode.toString()); + iterator.remove(); + } else if (Names.TREE_LEVELS.equals(fieldName)) { + builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); + iterator.remove(); + } else if (Names.TREE_PRESISION.equals(fieldName)) { + builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), + DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); + iterator.remove(); + } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { + builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); + iterator.remove(); + } else if (Names.ORIENTATION.equals(fieldName)) { + builder.fieldType().setOrientation(ShapeBuilder.Orientation.fromString(fieldNode.toString())); + iterator.remove(); + } else if (Names.STRATEGY.equals(fieldName)) { + builder.fieldType().setStrategyName(fieldNode.toString()); + iterator.remove(); + } else if (IGNORE_MALFORMED.equals(fieldName)) { + builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed")); + iterator.remove(); + } else if (Names.COERCE.equals(fieldName)) { + builder.coerce(XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE)); + iterator.remove(); + } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { + builder.ignoreZValue(XContentMapValues.nodeBooleanValue(fieldNode, + name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); + iterator.remove(); + } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) { + pointsOnly = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.STRATEGY_POINTS_ONLY); + iterator.remove(); + } + } + if (pointsOnly != null) { + if (builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) && pointsOnly == false) { + throw new IllegalArgumentException("points_only cannot be set to false for term strategy"); + } else { + builder.fieldType().setPointsOnly(pointsOnly); + } + } + return builder; } + } + + public static final class GeoShapeFieldType extends MappedFieldType { + + private String tree = Defaults.TREE; + private String strategyName = Defaults.STRATEGY; + private boolean pointsOnly = Defaults.POINTS_ONLY; + private int treeLevels = 0; + private double precisionInMeters = -1; + private Double distanceErrorPct; + private double defaultDistanceErrorPct = 0.0; + private Orientation orientation = Defaults.ORIENTATION; + + // these are built when the field type is frozen + private PrefixTreeStrategy defaultStrategy; + private RecursivePrefixTreeStrategy recursiveStrategy; + private TermQueryPrefixTreeStrategy termStrategy; + + public GeoShapeFieldType() {} protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); + this.tree = ref.tree; + this.strategyName = ref.strategyName; + this.pointsOnly = ref.pointsOnly; + this.treeLevels = ref.treeLevels; + this.precisionInMeters = ref.precisionInMeters; + this.distanceErrorPct = ref.distanceErrorPct; + this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; + this.orientation = ref.orientation; } @Override public GeoShapeFieldType clone() { return new GeoShapeFieldType(this); } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + GeoShapeFieldType that = (GeoShapeFieldType) o; + return treeLevels == that.treeLevels && + precisionInMeters == that.precisionInMeters && + defaultDistanceErrorPct == that.defaultDistanceErrorPct && + Objects.equals(tree, that.tree) && + Objects.equals(strategyName, that.strategyName) && + pointsOnly == that.pointsOnly && + Objects.equals(distanceErrorPct, that.distanceErrorPct) && + orientation == that.orientation; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tree, strategyName, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, + defaultDistanceErrorPct, orientation); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public void freeze() { + super.freeze(); + // This is a bit hackish: we need to setup the spatial tree and strategies once the field name is set, which + // must be by the time freeze is called. + SpatialPrefixTree prefixTree; + if ("geohash".equals(tree)) { + prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true)); + } else if ("legacyquadtree".equals(tree)) { + prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); + } else if ("quadtree".equals(tree)) { + prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + tree + "]"); + } + + recursiveStrategy = new RecursivePrefixTreeStrategy(prefixTree, name()); + recursiveStrategy.setDistErrPct(distanceErrorPct()); + recursiveStrategy.setPruneLeafyBranches(false); + termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); + termStrategy.setDistErrPct(distanceErrorPct()); + defaultStrategy = resolveStrategy(strategyName); + defaultStrategy.setPointsOnly(pointsOnly); + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts) { + super.checkCompatibility(fieldType, conflicts); + GeoShapeFieldType other = (GeoShapeFieldType)fieldType; + // prevent user from changing strategies + if (strategyName().equals(other.strategyName()) == false) { + conflicts.add("mapper [" + name() + "] has different [strategy]"); + } + + // prevent user from changing trees (changes encoding) + if (tree().equals(other.tree()) == false) { + conflicts.add("mapper [" + name() + "] has different [tree]"); + } + + if ((pointsOnly() != other.pointsOnly())) { + conflicts.add("mapper [" + name() + "] has different points_only"); + } + + // TODO we should allow this, but at the moment levels is used to build bookkeeping variables + // in lucene's SpatialPrefixTree implementations, need a patch to correct that first + if (treeLevels() != other.treeLevels()) { + conflicts.add("mapper [" + name() + "] has different [tree_levels]"); + } + if (precisionInMeters() != other.precisionInMeters()) { + conflicts.add("mapper [" + name() + "] has different [precision]"); + } + } + + private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { + if (treeLevels > 0 || precisionInMeters >= 0) { + return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) + : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); + } + return defaultLevels; + } + + public String tree() { + return tree; + } + + public void setTree(String tree) { + checkIfFrozen(); + this.tree = tree; + } + + public String strategyName() { + return strategyName; + } + + public void setStrategyName(String strategyName) { + checkIfFrozen(); + this.strategyName = strategyName; + if (this.strategyName.equals(SpatialStrategy.TERM.getStrategyName())) { + this.pointsOnly = true; + } + } + + public boolean pointsOnly() { + return pointsOnly; + } + + public void setPointsOnly(boolean pointsOnly) { + checkIfFrozen(); + this.pointsOnly = pointsOnly; + } + public int treeLevels() { + return treeLevels; + } + + public void setTreeLevels(int treeLevels) { + checkIfFrozen(); + this.treeLevels = treeLevels; + } + + public double precisionInMeters() { + return precisionInMeters; + } + + public void setPrecisionInMeters(double precisionInMeters) { + checkIfFrozen(); + this.precisionInMeters = precisionInMeters; + } + + public double distanceErrorPct() { + return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; + } + + public void setDistanceErrorPct(double distanceErrorPct) { + checkIfFrozen(); + this.distanceErrorPct = distanceErrorPct; + } + + public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { + checkIfFrozen(); + this.defaultDistanceErrorPct = defaultDistanceErrorPct; + } + + public Orientation orientation() { return this.orientation; } + + public void setOrientation(Orientation orientation) { + checkIfFrozen(); + this.orientation = orientation; + } + + public PrefixTreeStrategy defaultStrategy() { + return this.defaultStrategy; + } + + public PrefixTreeStrategy resolveStrategy(SpatialStrategy strategy) { + return resolveStrategy(strategy.getStrategyName()); + } + + public PrefixTreeStrategy resolveStrategy(String strategyName) { + if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { + return recursiveStrategy; + } + if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { + return termStrategy; + } + throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); + } } - public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + protected Explicit coerce; + protected Explicit ignoreMalformed; + protected Explicit ignoreZValue; + + public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit ignoreMalformed, + Explicit coerce, Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, - multiFields, copyTo); + super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); + this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; + this.ignoreZValue = ignoreZValue; } @Override public GeoShapeFieldType fieldType() { return (GeoShapeFieldType) super.fieldType(); } - - /** parsing logic for {@link LatLonShape} indexing */ @Override public void parse(ParseContext context) throws IOException { try { - Object shape = context.parseExternalValue(Object.class); + Shape shape = context.parseExternalValue(Shape.class); if (shape == null) { ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); if (shapeBuilder == null) { return; } - shape = shapeBuilder.buildLucene(); + shape = shapeBuilder.buildS4J(); + } + if (fieldType().pointsOnly() == true) { + // index configured for pointsOnly + if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { + // MULTIPOINT data: index each point separately + List shapes = ((XShapeCollection) shape).getShapes(); + for (Shape s : shapes) { + indexShape(context, s); + } + return; + } else if (shape instanceof Point == false) { + throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " + + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + + " was found"); + } } indexShape(context, shape); } catch (Exception e) { if (ignoreMalformed.value() == false) { throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), - fieldType().typeName()); - } - context.addIgnoredField(fieldType().name()); - } - } - - private void indexShape(ParseContext context, Object luceneShape) { - if (luceneShape instanceof GeoPoint) { - GeoPoint pt = (GeoPoint) luceneShape; - indexFields(context, LatLonShape.createIndexableFields(name(), pt.lat(), pt.lon())); - } else if (luceneShape instanceof double[]) { - double[] pt = (double[]) luceneShape; - indexFields(context, LatLonShape.createIndexableFields(name(), pt[1], pt[0])); - } else if (luceneShape instanceof Line) { - indexFields(context, LatLonShape.createIndexableFields(name(), (Line)luceneShape)); - } else if (luceneShape instanceof Polygon) { - indexFields(context, LatLonShape.createIndexableFields(name(), (Polygon) luceneShape)); - } else if (luceneShape instanceof double[][]) { - double[][] pts = (double[][])luceneShape; - for (int i = 0; i < pts.length; ++i) { - indexFields(context, LatLonShape.createIndexableFields(name(), pts[i][1], pts[i][0])); - } - } else if (luceneShape instanceof Line[]) { - Line[] lines = (Line[]) luceneShape; - for (int i = 0; i < lines.length; ++i) { - indexFields(context, LatLonShape.createIndexableFields(name(), lines[i])); - } - } else if (luceneShape instanceof Polygon[]) { - Polygon[] polys = (Polygon[]) luceneShape; - for (int i = 0; i < polys.length; ++i) { - indexFields(context, LatLonShape.createIndexableFields(name(), polys[i])); - } - } else if (luceneShape instanceof Rectangle) { - // index rectangle as a polygon - Rectangle r = (Rectangle) luceneShape; - Polygon p = new Polygon(new double[]{r.minLat, r.minLat, r.maxLat, r.maxLat, r.minLat}, - new double[]{r.minLon, r.maxLon, r.maxLon, r.minLon, r.minLon}); - indexFields(context, LatLonShape.createIndexableFields(name(), p)); - } else if (luceneShape instanceof Object[]) { - // recurse to index geometry collection - for (Object o : (Object[])luceneShape) { - indexShape(context, o); + fieldType().typeName()); } - } else { - throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass() + "] while indexing shape"); + context.addIgnoredField(fieldType.name()); + } + } + + private void indexShape(ParseContext context, Shape shape) { + List fields = new ArrayList<>(Arrays.asList(fieldType().defaultStrategy().createIndexableFields(shape))); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected void doMerge(Mapper mergeWith) { + super.doMerge(mergeWith); + + GeoShapeFieldMapper gsfm = (GeoShapeFieldMapper)mergeWith; + if (gsfm.coerce.explicit()) { + this.coerce = gsfm.coerce; + } + if (gsfm.ignoreMalformed.explicit()) { + this.ignoreMalformed = gsfm.ignoreMalformed; + } + if (gsfm.ignoreZValue.explicit()) { + this.ignoreZValue = gsfm.ignoreZValue; } } - private void indexFields(ParseContext context, Field[] fields) { - ArrayList flist = new ArrayList<>(Arrays.asList(fields)); - createFieldNamesField(context, flist); - for (IndexableField f : flist) { - context.doc().add(f); + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + builder.field("type", contentType()); + + if (includeDefaults || fieldType().tree().equals(Defaults.TREE) == false) { + builder.field(Names.TREE, fieldType().tree()); + } + + if (fieldType().treeLevels() != 0) { + builder.field(Names.TREE_LEVELS, fieldType().treeLevels()); + } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified + if ("geohash".equals(fieldType().tree())) { + builder.field(Names.TREE_LEVELS, Defaults.GEOHASH_LEVELS); + } else if ("legacyquadtree".equals(fieldType().tree())) { + builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); + } else if ("quadtree".equals(fieldType().tree())) { + builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); + } } + if (fieldType().precisionInMeters() != -1) { + builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(fieldType().precisionInMeters())); + } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified + builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(50)); + } + if (includeDefaults || fieldType().strategyName().equals(Defaults.STRATEGY) == false) { + builder.field(Names.STRATEGY, fieldType().strategyName()); + } + if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { + builder.field(Names.DISTANCE_ERROR_PCT, fieldType().distanceErrorPct()); + } + if (includeDefaults || fieldType().orientation() != Defaults.ORIENTATION) { + builder.field(Names.ORIENTATION, fieldType().orientation()); + } + if (fieldType().strategyName().equals(SpatialStrategy.TERM.getStrategyName())) { + // For TERMs strategy the defaults for points only change to true + if (includeDefaults || fieldType().pointsOnly() != true) { + builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); + } + } else { + if (includeDefaults || fieldType().pointsOnly() != GeoShapeFieldMapper.Defaults.POINTS_ONLY) { + builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); + } + } + if (includeDefaults || coerce.explicit()) { + builder.field(Names.COERCE, coerce.value()); + } + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); + } + if (includeDefaults || ignoreZValue.explicit()) { + builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); + } + } + + public Explicit coerce() { + return coerce; + } + + public Explicit ignoreMalformed() { + return ignoreMalformed; + } + + public Explicit ignoreZValue() { + return ignoreZValue; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java deleted file mode 100644 index b68e48305b24b..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.geo.ShapesAvailability; -import org.elasticsearch.common.geo.SpatialStrategy; -import org.elasticsearch.common.geo.XShapeCollection; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; -import org.elasticsearch.common.geo.parsers.ShapeParser; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; -import org.locationtech.spatial4j.shape.jts.JtsGeometry; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -/** - * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. - *

    - * Currently Shapes can only be indexed and can only be queried using - * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently - * a lot of behavior in this Mapper is disabled. - *

    - * Format supported: - *

    - * "field" : { - * "type" : "polygon", - * "coordinates" : [ - * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] - * ] - * } - *

    - * or: - *

    - * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) - * - * @deprecated use {@link GeoShapeFieldMapper} - */ -@Deprecated -public class LegacyGeoShapeFieldMapper extends BaseGeoShapeFieldMapper { - - public static final String CONTENT_TYPE = "geo_shape"; - - @Deprecated - public static class DeprecatedParameters { - public static class Names { - public static final ParseField STRATEGY = new ParseField("strategy"); - public static final ParseField TREE = new ParseField("tree"); - public static final ParseField TREE_LEVELS = new ParseField("tree_levels"); - public static final ParseField PRECISION = new ParseField("precision"); - public static final ParseField DISTANCE_ERROR_PCT = new ParseField("distance_error_pct"); - public static final ParseField POINTS_ONLY = new ParseField("points_only"); - } - - public static class PrefixTrees { - public static final String LEGACY_QUADTREE = "legacyquadtree"; - public static final String QUADTREE = "quadtree"; - public static final String GEOHASH = "geohash"; - } - - public static class Defaults { - public static final SpatialStrategy STRATEGY = SpatialStrategy.RECURSIVE; - public static final String TREE = "quadtree"; - public static final String PRECISION = "50m"; - public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision(PRECISION); - public static final int GEOHASH_TREE_LEVELS = GeoUtils.geoHashLevelsForPrecision(PRECISION); - public static final boolean POINTS_ONLY = false; - public static final double DISTANCE_ERROR_PCT = 0.025d; - } - - public SpatialStrategy strategy = null; - public String tree = null; - public int treeLevels = Integer.MIN_VALUE; - public String precision = null; - public Boolean pointsOnly = null; - public double distanceErrorPct = Double.NaN; - - public void setSpatialStrategy(SpatialStrategy strategy) { - this.strategy = strategy; - } - - public void setTree(String prefixTree) { - this.tree = prefixTree; - } - - public void setTreeLevels(int treeLevels) { - this.treeLevels = treeLevels; - } - - public void setPrecision(String precision) { - this.precision = precision; - } - - public void setPointsOnly(boolean pointsOnly) { - if (this.strategy == SpatialStrategy.TERM && pointsOnly == false) { - throw new ElasticsearchParseException("points_only cannot be set to false for term strategy"); - } - this.pointsOnly = pointsOnly; - } - - public void setDistanceErrorPct(double distanceErrorPct) { - this.distanceErrorPct = distanceErrorPct; - } - - protected void setup() { - if (strategy == null) { - strategy = Defaults.STRATEGY; - } - if (tree == null) { - tree = Defaults.TREE; - } - if (Double.isNaN(distanceErrorPct)) { - if (precision != null || treeLevels != Integer.MIN_VALUE) { - distanceErrorPct = 0d; - } else { - distanceErrorPct = Defaults.DISTANCE_ERROR_PCT; - } - } - if (treeLevels == Integer.MIN_VALUE && precision == null) { - // set default precision if treeLevels is not explicitly set - precision = Defaults.PRECISION; - } - if (treeLevels == Integer.MIN_VALUE) { - if (precision.equals(Defaults.PRECISION)) { - treeLevels = tree.equals(Defaults.TREE) - ? Defaults.QUADTREE_LEVELS - : Defaults.GEOHASH_TREE_LEVELS; - } else { - treeLevels = tree == Defaults.TREE - ? GeoUtils.quadTreeLevelsForPrecision(precision) - : GeoUtils.geoHashLevelsForPrecision(precision); - } - } - if (pointsOnly == null) { - if (strategy == SpatialStrategy.TERM) { - pointsOnly = true; - } else { - pointsOnly = Defaults.POINTS_ONLY; - } - } - } - - public static boolean parse(String name, String fieldName, Object fieldNode, DeprecatedParameters deprecatedParameters) { - if (Names.STRATEGY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setSpatialStrategy(SpatialStrategy.fromString(fieldNode.toString())); - } else if (Names.TREE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setTree(fieldNode.toString()); - } else if (Names.TREE_LEVELS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setTreeLevels(Integer.parseInt(fieldNode.toString())); - } else if (Names.PRECISION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setPrecision(fieldNode.toString()); - } else if (Names.DISTANCE_ERROR_PCT.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); - } else if (Names.POINTS_ONLY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - checkPrefixTreeSupport(fieldName); - deprecatedParameters.setPointsOnly( - XContentMapValues.nodeBooleanValue(fieldNode, name + "." + DeprecatedParameters.Names.POINTS_ONLY)); - } else { - return false; - } - return true; - } - - private static void checkPrefixTreeSupport(String fieldName) { - if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { - throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", - fieldName, CONTENT_TYPE); - } - DEPRECATION_LOGGER.deprecated("Field parameter [{}] is deprecated and will be removed in a future version.", - fieldName); - } - } - - private static final Logger logger = LogManager.getLogger(LegacyGeoShapeFieldMapper.class); - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); - - public static class Builder extends BaseGeoShapeFieldMapper.Builder { - - DeprecatedParameters deprecatedParameters; - - public Builder(String name) { - super(name, new GeoShapeFieldType(), new GeoShapeFieldType()); - this.deprecatedParameters = new DeprecatedParameters(); - this.deprecatedParameters.setup(); - } - - public Builder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, - boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); - this.deprecatedParameters = deprecatedParameters; - this.deprecatedParameters.setup(); - } - - @Override - public GeoShapeFieldType fieldType() { - return (GeoShapeFieldType)fieldType; - } - - private void setupFieldTypeDeprecatedParameters() { - GeoShapeFieldType ft = fieldType(); - ft.setStrategy(deprecatedParameters.strategy); - ft.setTree(deprecatedParameters.tree); - ft.setTreeLevels(deprecatedParameters.treeLevels); - if (deprecatedParameters.precision != null) { - // precision is only set iff: a. treeLevel is not explicitly set, b. its explicitly set - ft.setPrecisionInMeters(DistanceUnit.parse(deprecatedParameters.precision, - DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); - } - ft.setDistanceErrorPct(deprecatedParameters.distanceErrorPct); - ft.setPointsOnly(deprecatedParameters.pointsOnly); - } - - private void setupPrefixTrees() { - GeoShapeFieldType ft = fieldType(); - SpatialPrefixTree prefixTree; - if (ft.tree().equals(DeprecatedParameters.PrefixTrees.GEOHASH)) { - prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS, true)); - } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE)) { - prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); - } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.QUADTREE)) { - prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + ft.tree() + "]"); - } - - // setup prefix trees regardless of strategy (this is used for the QueryBuilder) - // recursive: - RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, ft.name()); - rpts.setDistErrPct(ft.distanceErrorPct()); - rpts.setPruneLeafyBranches(false); - ft.recursiveStrategy = rpts; - - // term: - TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, ft.name()); - termStrategy.setDistErrPct(ft.distanceErrorPct()); - ft.termStrategy = termStrategy; - - // set default (based on strategy): - ft.defaultPrefixTreeStrategy = ft.resolvePrefixTreeStrategy(ft.strategy()); - ft.defaultPrefixTreeStrategy.setPointsOnly(ft.pointsOnly()); - } - - @Override - protected void setupFieldType(BuilderContext context) { - super.setupFieldType(context); - - // field mapper handles this at build time - // but prefix tree strategies require a name, so throw a similar exception - if (fieldType().name().isEmpty()) { - throw new IllegalArgumentException("name cannot be empty string"); - } - - // setup the deprecated parameters and the prefix tree configuration - setupFieldTypeDeprecatedParameters(); - setupPrefixTrees(); - } - - private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { - if (treeLevels > 0 || precisionInMeters >= 0) { - return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) - : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); - } - return defaultLevels; - } - - @Override - public LegacyGeoShapeFieldMapper build(BuilderContext context) { - setupFieldType(context); - - return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), - coerce(context), orientation(), ignoreZValue(), context.indexSettings(), - multiFieldsBuilder.build(this, context), copyTo); - } - } - - public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { - - private String tree = DeprecatedParameters.Defaults.TREE; - private SpatialStrategy strategy = DeprecatedParameters.Defaults.STRATEGY; - private boolean pointsOnly = DeprecatedParameters.Defaults.POINTS_ONLY; - private int treeLevels = 0; - private double precisionInMeters = -1; - private Double distanceErrorPct; - private double defaultDistanceErrorPct = 0.0; - - // these are built when the field type is frozen - private PrefixTreeStrategy defaultPrefixTreeStrategy; - private RecursivePrefixTreeStrategy recursiveStrategy; - private TermQueryPrefixTreeStrategy termStrategy; - - public GeoShapeFieldType() { - setIndexOptions(IndexOptions.DOCS); - setTokenized(false); - setStored(false); - setStoreTermVectors(false); - setOmitNorms(true); - } - - protected GeoShapeFieldType(GeoShapeFieldType ref) { - super(ref); - this.tree = ref.tree; - this.strategy = ref.strategy; - this.pointsOnly = ref.pointsOnly; - this.treeLevels = ref.treeLevels; - this.precisionInMeters = ref.precisionInMeters; - this.distanceErrorPct = ref.distanceErrorPct; - this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; - } - - @Override - public GeoShapeFieldType clone() { - return new GeoShapeFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - GeoShapeFieldType that = (GeoShapeFieldType) o; - return treeLevels == that.treeLevels && - precisionInMeters == that.precisionInMeters && - defaultDistanceErrorPct == that.defaultDistanceErrorPct && - Objects.equals(tree, that.tree) && - Objects.equals(strategy, that.strategy) && - pointsOnly == that.pointsOnly && - Objects.equals(distanceErrorPct, that.distanceErrorPct); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, - defaultDistanceErrorPct); - } - - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts) { - super.checkCompatibility(fieldType, conflicts); - GeoShapeFieldType other = (GeoShapeFieldType)fieldType; - // prevent user from changing strategies - if (strategy() != other.strategy()) { - conflicts.add("mapper [" + name() + "] has different [strategy]"); - } - - // prevent user from changing trees (changes encoding) - if (tree().equals(other.tree()) == false) { - conflicts.add("mapper [" + name() + "] has different [tree]"); - } - - if ((pointsOnly() != other.pointsOnly())) { - conflicts.add("mapper [" + name() + "] has different points_only"); - } - - // TODO we should allow this, but at the moment levels is used to build bookkeeping variables - // in lucene's SpatialPrefixTree implementations, need a patch to correct that first - if (treeLevels() != other.treeLevels()) { - conflicts.add("mapper [" + name() + "] has different [tree_levels]"); - } - if (precisionInMeters() != other.precisionInMeters()) { - conflicts.add("mapper [" + name() + "] has different [precision]"); - } - } - - public String tree() { - return tree; - } - - public void setTree(String tree) { - checkIfFrozen(); - this.tree = tree; - } - - public SpatialStrategy strategy() { - return strategy; - } - - public void setStrategy(SpatialStrategy strategy) { - checkIfFrozen(); - this.strategy = strategy; - if (this.strategy.equals(SpatialStrategy.TERM)) { - this.pointsOnly = true; - } - } - - public boolean pointsOnly() { - return pointsOnly; - } - - public void setPointsOnly(boolean pointsOnly) { - checkIfFrozen(); - this.pointsOnly = pointsOnly; - } - public int treeLevels() { - return treeLevels; - } - - public void setTreeLevels(int treeLevels) { - checkIfFrozen(); - this.treeLevels = treeLevels; - } - - public double precisionInMeters() { - return precisionInMeters; - } - - public void setPrecisionInMeters(double precisionInMeters) { - checkIfFrozen(); - this.precisionInMeters = precisionInMeters; - } - - public double distanceErrorPct() { - return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; - } - - public void setDistanceErrorPct(double distanceErrorPct) { - checkIfFrozen(); - this.distanceErrorPct = distanceErrorPct; - } - - public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { - checkIfFrozen(); - this.defaultDistanceErrorPct = defaultDistanceErrorPct; - } - - public PrefixTreeStrategy defaultPrefixTreeStrategy() { - return this.defaultPrefixTreeStrategy; - } - - public PrefixTreeStrategy resolvePrefixTreeStrategy(SpatialStrategy strategy) { - return resolvePrefixTreeStrategy(strategy.getStrategyName()); - } - - public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { - if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { - return recursiveStrategy; - } - if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { - return termStrategy; - } - throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); - } - } - - public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, Explicit orientation, - Explicit ignoreZValue, Settings indexSettings, - MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, - multiFields, copyTo); - } - - @Override - public GeoShapeFieldType fieldType() { - return (GeoShapeFieldType) super.fieldType(); - } - - @Override - public void parse(ParseContext context) throws IOException { - try { - Shape shape = context.parseExternalValue(Shape.class); - if (shape == null) { - ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); - if (shapeBuilder == null) { - return; - } - shape = shapeBuilder.buildS4J(); - } - if (fieldType().pointsOnly() == true) { - // index configured for pointsOnly - if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { - // MULTIPOINT data: index each point separately - List shapes = ((XShapeCollection) shape).getShapes(); - for (Shape s : shapes) { - indexShape(context, s); - } - return; - } else if (shape instanceof Point == false) { - throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " - + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) - + " was found"); - } - } - indexShape(context, shape); - } catch (Exception e) { - if (ignoreMalformed.value() == false) { - throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), - fieldType().typeName()); - } - context.addIgnoredField(fieldType.name()); - } - } - - private void indexShape(ParseContext context, Shape shape) { - List fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape))); - createFieldNamesField(context, fields); - for (IndexableField field : fields) { - context.doc().add(field); - } - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - - if (includeDefaults || fieldType().tree().equals(DeprecatedParameters.Defaults.TREE) == false) { - builder.field(DeprecatedParameters.Names.TREE.getPreferredName(), fieldType().tree()); - } - - if (fieldType().treeLevels() != 0) { - builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), fieldType().treeLevels()); - } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified - if (DeprecatedParameters.PrefixTrees.GEOHASH.equals(fieldType().tree())) { - builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), - DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS); - } else if (DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE.equals(fieldType().tree())) { - builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), - DeprecatedParameters.Defaults.QUADTREE_LEVELS); - } else if (DeprecatedParameters.PrefixTrees.QUADTREE.equals(fieldType().tree())) { - builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), - DeprecatedParameters.Defaults.QUADTREE_LEVELS); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); - } - } - if (fieldType().precisionInMeters() != -1) { - builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), - DistanceUnit.METERS.toString(fieldType().precisionInMeters())); - } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified - builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), - DistanceUnit.METERS.toString(50)); - } - - builder.field(DeprecatedParameters.Names.STRATEGY.getPreferredName(), fieldType().strategy().getStrategyName()); - - if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { - builder.field(DeprecatedParameters.Names.DISTANCE_ERROR_PCT.getPreferredName(), fieldType().distanceErrorPct()); - } - if (fieldType().strategy() == SpatialStrategy.TERM) { - // For TERMs strategy the defaults for points only change to true - if (includeDefaults || fieldType().pointsOnly() != true) { - builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); - } - } else { - if (includeDefaults || fieldType().pointsOnly() != DeprecatedParameters.Defaults.POINTS_ONLY) { - builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); - } - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index 6ee0f3f10ddcc..c517050896946 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -19,10 +19,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.document.LatLonShape; -import org.apache.lucene.geo.Line; -import org.apache.lucene.geo.Polygon; -import org.apache.lucene.geo.Rectangle; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -40,9 +36,8 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.parsers.ShapeParser; @@ -53,8 +48,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; @@ -335,9 +329,9 @@ public GeoShapeQueryBuilder relation(ShapeRelation relation) { if (relation == null) { throw new IllegalArgumentException("No Shape Relation defined"); } - if (SpatialStrategy.TERM.equals(strategy) && relation != ShapeRelation.INTERSECTS) { + if (strategy != null && strategy == SpatialStrategy.TERM && relation != ShapeRelation.INTERSECTS) { throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" - + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); + + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); } this.relation = relation; return this; @@ -382,96 +376,32 @@ protected Query doToQuery(QueryShardContext context) { } else { throw new QueryShardException(context, "failed to find geo_shape field [" + fieldName + "]"); } - } else if (fieldType.typeName().equals(BaseGeoShapeFieldMapper.CONTENT_TYPE) == false) { + } else if (fieldType.typeName().equals(GeoShapeFieldMapper.CONTENT_TYPE) == false) { throw new QueryShardException(context, "Field [" + fieldName + "] is not of type [geo_shape] but of type [" + fieldType.typeName() + "]"); } - final BaseGeoShapeFieldMapper.BaseGeoShapeFieldType ft = (BaseGeoShapeFieldMapper.BaseGeoShapeFieldType) fieldType; - Query query; - if (strategy != null || ft instanceof LegacyGeoShapeFieldMapper.GeoShapeFieldType) { - LegacyGeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (LegacyGeoShapeFieldMapper.GeoShapeFieldType) ft; - SpatialStrategy spatialStrategy = shapeFieldType.strategy(); - if (this.strategy != null) { - spatialStrategy = this.strategy; - } - PrefixTreeStrategy prefixTreeStrategy = shapeFieldType.resolvePrefixTreeStrategy(spatialStrategy); - if (prefixTreeStrategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { - // this strategy doesn't support disjoint anymore: but it did - // before, including creating lucene fieldcache (!) - // in this case, execute disjoint as exists && !intersects - BooleanQuery.Builder bool = new BooleanQuery.Builder(); - Query exists = ExistsQueryBuilder.newFilter(context, fieldName); - Query intersects = prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); - bool.add(exists, BooleanClause.Occur.MUST); - bool.add(intersects, BooleanClause.Occur.MUST_NOT); - query = new ConstantScoreQuery(bool.build()); - } else { - query = new ConstantScoreQuery(prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, relation))); - } - } else { - query = new ConstantScoreQuery(getVectorQuery(context, shapeToQuery)); - } - return query; - } + final GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType; - private Query getVectorQuery(QueryShardContext context, ShapeBuilder queryShapeBuilder) { - // CONTAINS queries are not yet supported by VECTOR strategy - if (relation == ShapeRelation.CONTAINS) { - throw new QueryShardException(context, - ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "]"); + PrefixTreeStrategy strategy = shapeFieldType.defaultStrategy(); + if (this.strategy != null) { + strategy = shapeFieldType.resolveStrategy(this.strategy); } - - // wrap geoQuery as a ConstantScoreQuery - return getVectorQueryFromShape(context, queryShapeBuilder.buildLucene()); - } - - private Query getVectorQueryFromShape(QueryShardContext context, Object queryShape) { - Query geoQuery; - if (queryShape instanceof Line[]) { - geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line[]) queryShape); - } else if (queryShape instanceof Polygon[]) { - geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon[]) queryShape); - } else if (queryShape instanceof Line) { - geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line) queryShape); - } else if (queryShape instanceof Polygon) { - geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon) queryShape); - } else if (queryShape instanceof Rectangle) { - Rectangle r = (Rectangle) queryShape; - geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), - r.minLat, r.maxLat, r.minLon, r.maxLon); - } else if (queryShape instanceof double[][]) { - // note: we decompose point queries into a bounding box query with min values == max values - // to do this for multipoint we would have to create a BooleanQuery for each point - // this is *way* too costly. So we do not allow multipoint queries - throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.MULTIPOINT + " queries"); - } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { - // for now just create a single bounding box query with min values == max values - double[] pt; - if (queryShape instanceof GeoPoint) { - pt = new double[] {((GeoPoint)queryShape).lon(), ((GeoPoint)queryShape).lat()}; - } else { - pt = (double[])queryShape; - if (pt.length != 2) { - throw new QueryShardException(context, "Expected double array of length 2. " - + "But found length " + pt.length + " for field [" + fieldName + "]"); - } - } - return LatLonShape.newBoxQuery(fieldName, relation.getLuceneRelation(), pt[1], pt[1], pt[0], pt[0]); - } else if (queryShape instanceof Object[]) { - geoQuery = createGeometryCollectionQuery(context, (Object[]) queryShape); + Query query; + if (strategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { + // this strategy doesn't support disjoint anymore: but it did + // before, including creating lucene fieldcache (!) + // in this case, execute disjoint as exists && !intersects + BooleanQuery.Builder bool = new BooleanQuery.Builder(); + Query exists = ExistsQueryBuilder.newFilter(context, fieldName); + Query intersects = strategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); + bool.add(exists, BooleanClause.Occur.MUST); + bool.add(intersects, BooleanClause.Occur.MUST_NOT); + query = new ConstantScoreQuery(bool.build()); } else { - throw new QueryShardException(context, "Field [" + fieldName + "] found and unknown shape"); + query = new ConstantScoreQuery(strategy.makeQuery(getArgs(shapeToQuery, relation))); } - return geoQuery; - } - - private Query createGeometryCollectionQuery(QueryShardContext context, Object... shapes) { - BooleanQuery.Builder bqb = new BooleanQuery.Builder(); - for (Object shape : shapes) { - bqb.add(getVectorQueryFromShape(context, shape), BooleanClause.Occur.SHOULD); - } - return bqb.build(); + return query; } /** @@ -484,6 +414,9 @@ private Query createGeometryCollectionQuery(QueryShardContext context, Object... * Shape itself is located */ private void fetch(Client client, GetRequest getRequest, String path, ActionListener listener) { + if (ShapesAvailability.JTS_AVAILABLE == false) { + throw new IllegalStateException("JTS not available"); + } getRequest.preference("_local"); client.get(getRequest, new ActionListener(){ diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 24b5d7f427ca2..a1038853c0670 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -25,13 +25,13 @@ import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -131,7 +132,10 @@ private Map getMappers(List mapperPlugi mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - mappers.put(BaseGeoShapeFieldMapper.CONTENT_TYPE, new BaseGeoShapeFieldMapper.TypeParser()); + + if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { + mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); + } for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index 2acabee8797f4..a9a210549064f 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions; @@ -296,8 +296,7 @@ public void testParse3DPolygon() throws IOException { LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper mapperBuilder = - (LegacyGeoShapeFieldMapper) (new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); + final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); @@ -897,6 +896,7 @@ public void testParseMultiPoint() throws IOException { .startArray().value(101.0).value(1.0).endArray() .endArray() .endObject(); + ShapeCollection expected = shapeCollection( SPATIAL_CONTEXT.makePoint(100, 0), SPATIAL_CONTEXT.makePoint(101, 1.0)); @@ -968,6 +968,7 @@ public void testParseMultiPolygon() throws IOException { shellCoordinates.add(new Coordinate(102, 2)); shellCoordinates.add(new Coordinate(102, 3)); + shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); Polygon withoutHoles = GEOMETRY_FACTORY.createPolygon(shell, null); @@ -1148,6 +1149,7 @@ public void testThatParserExtractsCorrectTypeAndCoordinatesFromArbitraryJson() t .startObject("nested").startArray("coordinates").value(200.0).value(0.0).endArray().endObject() .startObject("lala").field("type", "NotAPoint").endObject() .endObject(); + Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson, true); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 94c96e00d9236..1b4c0b9dce048 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.locationtech.jts.geom.Coordinate; @@ -147,6 +146,7 @@ private List randomLineStringCoords() { @Override public void testParseLineString() throws IOException { List coordinates = randomLineStringCoords(); + LineString expected = GEOMETRY_FACTORY.createLineString(coordinates.toArray(new Coordinate[coordinates.size()])); assertExpected(jtsGeom(expected), new LineStringBuilder(coordinates), true); @@ -279,14 +279,13 @@ public void testParseMixedDimensionPolyWithHole() throws IOException { parser.nextToken(); Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_7_0_0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = - (GeoShapeFieldMapper) (new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext)); + final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext); // test store z disabled ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, @@ -324,8 +323,7 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper mapperBuilder = - (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); + final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); // test store z disabled ElasticsearchException e = expectThrows(ElasticsearchException.class, @@ -354,8 +352,7 @@ public void testParsePolyWithStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper mapperBuilder = - (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); + final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, mapperBuilder); assertEquals(shapeBuilder.numDimensions(), 3); @@ -375,14 +372,12 @@ public void testParseOpenPolygon() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper defaultMapperBuilder = - (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext)); + final GeoShapeFieldMapper defaultMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext); ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> ShapeParser.parse(parser, defaultMapperBuilder)); assertEquals("invalid LinearRing found (coordinates are not closed)", exception.getMessage()); - final LegacyGeoShapeFieldMapper coercingMapperBuilder = - (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext)); + final GeoShapeFieldMapper coercingMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, coercingMapperBuilder); assertNotNull(shapeBuilder); assertEquals("polygon ((100.0 5.0, 100.0 10.0, 90.0 10.0, 90.0 5.0, 100.0 5.0))", shapeBuilder.toWKT()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 20c49c00935e3..0e6854c41e3c4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -24,8 +24,8 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.elasticsearch.Version; import org.elasticsearch.common.geo.builders.PointBuilder; +import org.locationtech.spatial4j.shape.Point; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; @@ -63,7 +63,6 @@ public static class Builder extends FieldMapper.Builder private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); private GeoPointFieldMapper.Builder latLonPointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT); private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); - private LegacyGeoShapeFieldMapper.Builder legacyShapeBuilder = new LegacyGeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); private Mapper.Builder stringBuilder; private String generatedValue; private String mapperName; @@ -87,9 +86,7 @@ public ExternalMapper build(BuilderContext context) { BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = latLonPointBuilder.build(context); - BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_6_0)) - ? legacyShapeBuilder.build(context) - : shapeBuilder.build(context); + GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); context.path().remove(); @@ -153,13 +150,13 @@ public Query existsQuery(QueryShardContext context) { private BinaryFieldMapper binMapper; private BooleanFieldMapper boolMapper; private GeoPointFieldMapper pointMapper; - private BaseGeoShapeFieldMapper shapeMapper; + private GeoShapeFieldMapper shapeMapper; private FieldMapper stringMapper; public ExternalMapper(String simpleName, MappedFieldType fieldType, String generatedValue, String mapperName, BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, - BaseGeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, + GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, new ExternalFieldType(), indexSettings, multiFields, copyTo); this.generatedValue = generatedValue; @@ -185,12 +182,8 @@ public void parse(ParseContext context) throws IOException { pointMapper.parse(context.createExternalValueContext(point)); // Let's add a Dummy Shape - PointBuilder pb = new PointBuilder(-100, 45); - if (shapeMapper instanceof GeoShapeFieldMapper) { - shapeMapper.parse(context.createExternalValueContext(pb.buildLucene())); - } else { - shapeMapper.parse(context.createExternalValueContext(pb.buildS4J())); - } + Point shape = new PointBuilder(-100, 45).buildS4J(); + shapeMapper.parse(context.createExternalValueContext(shape)); context = context.createExternalValueContext(generatedValue); @@ -217,7 +210,7 @@ public FieldMapper updateFieldType(Map fullNameToFieldT BinaryFieldMapper binMapperUpdate = (BinaryFieldMapper) binMapper.updateFieldType(fullNameToFieldType); BooleanFieldMapper boolMapperUpdate = (BooleanFieldMapper) boolMapper.updateFieldType(fullNameToFieldType); GeoPointFieldMapper pointMapperUpdate = (GeoPointFieldMapper) pointMapper.updateFieldType(fullNameToFieldType); - BaseGeoShapeFieldMapper shapeMapperUpdate = (BaseGeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); + GeoShapeFieldMapper shapeMapperUpdate = (GeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); TextFieldMapper stringMapperUpdate = (TextFieldMapper) stringMapper.updateFieldType(fullNameToFieldType); if (update == this && multiFieldsUpdate == multiFields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java index 6d47e4a784e06..e1158f77bd47b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java @@ -21,13 +21,12 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.builders.EnvelopeBuilder; +import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.locationtech.jts.geom.Coordinate; import java.util.Arrays; import java.util.Collection; @@ -119,8 +118,7 @@ public void testExternalValues() throws Exception { assertThat(response.getHits().getTotalHits().value, equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", - new EnvelopeBuilder(new Coordinate(-101, 46), new Coordinate(-99, 44))).relation(ShapeRelation.WITHIN)) + .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", new PointBuilder(-100, 45)).relation(ShapeRelation.WITHIN)) .execute().actionGet(); assertThat(response.getHits().getTotalHits().value, equalTo((long) 1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index a5e2d7c31afe2..20e689e9d7e89 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -18,9 +18,14 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -37,6 +42,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { @@ -47,10 +53,10 @@ protected Collection> getPlugins() { public void testDefaultConfiguration() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -58,8 +64,12 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - assertThat(geoShapeFieldMapper.fieldType().orientation(), - equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.025d)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoShapeFieldMapper.Defaults.GEOHASH_LEVELS)); + assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION)); } /** @@ -67,11 +77,11 @@ public void testDefaultConfiguration() throws IOException { */ public void testOrientationParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -85,11 +95,11 @@ public void testOrientationParsing() throws IOException { // explicit right orientation test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -107,11 +117,11 @@ public void testOrientationParsing() throws IOException { */ public void testCoerceParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "true") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "true") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -123,11 +133,11 @@ public void testCoerceParsing() throws IOException { // explicit false coerce test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "false") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "false") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -136,7 +146,6 @@ public void testCoerceParsing() throws IOException { coerce = ((GeoShapeFieldMapper)fieldMapper).coerce().value(); assertThat(coerce, equalTo(false)); - assertFieldWarnings("tree"); } @@ -213,45 +222,304 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } + public void testGeohashConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "4") + .field("distance_error_pct", "0.1") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.1)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); + } + + public void testQuadtreeConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("distance_error_pct", "0.5") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + } + + public void testLevelPrecisionConfiguration() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "26") + .field("precision", "70m") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and + // "tree_levels" setting distErrPct to 0 to guarantee desired precision + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is less precise so it loses + assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); + } + } + + public void testPointsOnlyOption() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + } + + public void testLevelDefaults() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + - private void assertFieldWarnings(String... fieldNames) { - String[] warnings = new String[fieldNames.length]; - for (int i = 0; i < fieldNames.length; ++i) { - warnings[i] = "Field parameter [" + fieldNames[i] + "] " - + "is deprecated and will be removed in a future version."; + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); } } public void testGeoShapeMapperMerge() throws Exception { String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("shape").field("type", "geo_shape") - .field("orientation", "ccw") - .endObject().endObject().endObject().endObject()); + .startObject("shape").field("type", "geo_shape").field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) + .field("orientation", "ccw") + .endObject().endObject().endObject().endObject()); MapperService mapperService = createIndex("test").mapperService(); DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), MapperService.MergeReason.MAPPING_UPDATE); String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("orientation", "cw").endObject().endObject().endObject().endObject()); - mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "quadtree") + .field("strategy", "term").field("precision", "1km") + .field("tree_levels", 26).field("distance_error_pct", 26) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); + } // verify nothing changed Mapper fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.01)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); - // change mapping; orientation + // correct mapping stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("orientation", "cw").endObject().endObject().endObject().endObject()); + .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m") + .field("tree_levels", 8).field("distance_error_pct", 0.001) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.001)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); } @@ -276,12 +544,112 @@ public void testSerializeDefaults() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":9")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertFalse(serialized, serialized.contains("\"precision\":")); + assertTrue(serialized, serialized.contains("\"tree_levels\":6")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6") .endObject().endObject() .endObject().endObject()); DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"orientation\":\"" + BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertFalse(serialized, serialized.contains("\"tree_levels\":")); } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6m") + .field("tree_levels", "5") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":5")); + } + } + + public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + // term strategy changes the default for points_only, check that we handle it correctly + assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); + } + + + public void testPointsOnlyFalseWithTermStrategy() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .field("points_only", false) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> parser.parse("type1", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); } public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java index c10ec5facf806..a1c225f8a0657 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java @@ -18,23 +18,69 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.index.mapper.GeoShapeFieldMapper.GeoShapeFieldType; import org.junit.Before; +import java.io.IOException; + public class GeoShapeFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { - return new GeoShapeFieldType(); + return new GeoShapeFieldMapper.GeoShapeFieldType(); } @Before public void setupProperties() { - addModifier(new FieldTypeTestCase.Modifier("orientation", true) { + addModifier(new Modifier("tree", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTree("quadtree"); + } + }); + addModifier(new Modifier("strategy", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setStrategyName("term"); + } + }); + addModifier(new Modifier("tree_levels", false) { @Override public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTreeLevels(10); } }); + addModifier(new Modifier("precision", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setPrecisionInMeters(20); + } + }); + addModifier(new Modifier("distance_error_pct", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); + } + }); + addModifier(new Modifier("orientation", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); + } + }); + } + + /** + * Test for {@link GeoShapeFieldType#setStrategyName(String)} that checks that {@link GeoShapeFieldType#pointsOnly()} + * gets set as a side effect when using SpatialStrategy.TERM + */ + public void testSetStrategyName() throws IOException { + GeoShapeFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategyName(SpatialStrategy.RECURSIVE.getStrategyName()); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategyName(SpatialStrategy.TERM.getStrategyName()); + assertTrue(fieldType.pointsOnly()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java deleted file mode 100644 index 11d8c72531db2..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ /dev/null @@ -1,714 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.not; - -public class LegacyGeoShapeFieldMapperTests extends ESSingleNodeTestCase { - - @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class); - } - - public void testDefaultConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("strategy", "recursive") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - assertThat(geoShapeFieldMapper.fieldType().tree(), - equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.TREE)); - assertThat(geoShapeFieldMapper.fieldType().treeLevels(), - equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.QUADTREE_LEVELS)); - assertThat(geoShapeFieldMapper.fieldType().pointsOnly(), - equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.POINTS_ONLY)); - assertThat(geoShapeFieldMapper.fieldType().distanceErrorPct(), - equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.DISTANCE_ERROR_PCT)); - assertThat(geoShapeFieldMapper.fieldType().orientation(), - equalTo(LegacyGeoShapeFieldMapper.Defaults.ORIENTATION.value())); - assertFieldWarnings("strategy"); - } - - /** - * Test that orientation parameter correctly parses - */ - public void testOrientationParsing() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - ShapeBuilder.Orientation orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); - - // explicit right orientation test - mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); - - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); - assertFieldWarnings("tree"); - } - - /** - * Test that coerce parameter correctly parses - */ - public void testCoerceParsing() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("coerce", "true") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - boolean coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); - assertThat(coerce, equalTo(true)); - - // explicit false coerce test - mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("coerce", "false") - .endObject().endObject() - .endObject().endObject()); - - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); - assertThat(coerce, equalTo(false)); - assertFieldWarnings("tree"); - } - - - /** - * Test that accept_z_value parameter correctly parses - */ - public void testIgnoreZValue() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("strategy", "recursive") - .field(IGNORE_Z_VALUE.getPreferredName(), "true") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - boolean ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); - assertThat(ignoreZValue, equalTo(true)); - - // explicit false accept_z_value test - mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field(IGNORE_Z_VALUE.getPreferredName(), "false") - .endObject().endObject() - .endObject().endObject()); - - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); - assertThat(ignoreZValue, equalTo(false)); - assertFieldWarnings("strategy", "tree"); - } - - /** - * Test that ignore_malformed parameter correctly parses - */ - public void testIgnoreMalformedParsing() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("ignore_malformed", "true") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - Explicit ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); - assertThat(ignoreMalformed.value(), equalTo(true)); - - // explicit false ignore_malformed test - mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("ignore_malformed", "false") - .endObject().endObject() - .endObject().endObject()); - - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); - assertThat(ignoreMalformed.explicit(), equalTo(true)); - assertThat(ignoreMalformed.value(), equalTo(false)); - assertFieldWarnings("tree"); - } - - public void testGeohashConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "4") - .field("distance_error_pct", "0.1") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.1)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); - assertFieldWarnings("tree", "tree_levels", "distance_error_pct"); - } - - public void testQuadtreeConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("distance_error_pct", "0.5") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - assertFieldWarnings("tree", "tree_levels", "distance_error_pct", "points_only"); - } - - private void assertFieldWarnings(String... fieldNames) { - String[] warnings = new String[fieldNames.length]; - for (int i = 0; i < fieldNames.length; ++i) { - warnings[i] = "Field parameter [" + fieldNames[i] + "] " - + "is deprecated and will be removed in a future version."; - } - assertWarnings(warnings); - } - - public void testLevelPrecisionConfiguration() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "26") - .field("precision", "70m") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and - // "tree_levels" setting distErrPct to 0 to guarantee desired precision - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is less precise so it loses - assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); - } - assertFieldWarnings("tree", "tree_levels", "precision", "distance_error_pct"); - } - - public void testPointsOnlyOption() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - assertFieldWarnings("tree", "points_only"); - } - - public void testLevelDefaults() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); - } - assertFieldWarnings("tree", "distance_error_pct"); - } - - public void testGeoShapeMapperMerge() throws Exception { - String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("shape").field("type", "geo_shape").field("tree", "geohash") - .field("strategy", "recursive") - .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) - .field("orientation", "ccw") - .endObject().endObject().endObject().endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), - MapperService.MergeReason.MAPPING_UPDATE); - String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("tree", "quadtree") - .field("strategy", "term").field("precision", "1km") - .field("tree_levels", 26).field("distance_error_pct", 26) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); - try { - mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); - fail(); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); - } - - // verify nothing changed - Mapper fieldMapper = docMapper.mappers().getMapper("shape"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.01)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); - assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); - - // correct mapping - stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("tree", "geohash") - .field("strategy", "recursive") - .field("precision", "1m") - .field("tree_levels", 8).field("distance_error_pct", 0.001) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); - docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); - - fieldMapper = docMapper.mappers().getMapper("shape"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.001)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); - assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); - - assertFieldWarnings("tree", "strategy", "precision", "tree_levels", "distance_error_pct"); - } - - public void testEmptyName() throws Exception { - // after 5.x - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) - ); - assertThat(e.getMessage(), containsString("name cannot be empty string")); - assertFieldWarnings("tree"); - } - - public void testSerializeDefaults() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":21")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":9")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertFalse(serialized, serialized.contains("\"precision\":")); - assertTrue(serialized, serialized.contains("\"tree_levels\":6")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":10")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6m") - .field("tree_levels", "5") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":5")); - } - assertFieldWarnings("tree", "tree_levels", "precision"); - } - - public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); - - LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - // term strategy changes the default for points_only, check that we handle it correctly - assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); - assertFieldWarnings("tree", "precision", "strategy"); - } - - - public void testPointsOnlyFalseWithTermStrategy() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .field("points_only", false) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) - ); - assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); - assertFieldWarnings("tree", "precision", "strategy", "points_only"); - } - - public String toXContentString(LegacyGeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - ToXContent.Params params; - if (includeDefaults) { - params = new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true")); - } else { - params = ToXContent.EMPTY_PARAMS; - } - mapper.doXContentBody(builder, includeDefaults, params); - return Strings.toString(builder.endObject()); - } - - public String toXContentString(LegacyGeoShapeFieldMapper mapper) throws IOException { - return toXContentString(mapper, true); - } - -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java deleted file mode 100644 index 2fcbed82e33b4..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.geo.SpatialStrategy; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType; -import org.junit.Before; - -import java.io.IOException; - -public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType() { - return new GeoShapeFieldType(); - } - - @Before - public void setupProperties() { - addModifier(new Modifier("tree", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setTree("geohash"); - } - }); - addModifier(new Modifier("strategy", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setStrategy(SpatialStrategy.TERM); - } - }); - addModifier(new Modifier("tree_levels", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setTreeLevels(10); - } - }); - addModifier(new Modifier("precision", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setPrecisionInMeters(20); - } - }); - addModifier(new Modifier("distance_error_pct", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); - } - }); - addModifier(new Modifier("orientation", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); - } - }); - } - - /** - * Test for {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks - * that {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#pointsOnly()} gets set as a side effect when using SpatialStrategy.TERM - */ - public void testSetStrategyName() throws IOException { - GeoShapeFieldType fieldType = new GeoShapeFieldType(); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategy(SpatialStrategy.RECURSIVE); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategy(SpatialStrategy.TERM); - assertTrue(fieldType.pointsOnly()); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index e2e4db1f9b790..bcd2b4ef14440 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanQuery; @@ -28,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -52,41 +54,29 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase { - protected static String indexedShapeId; - protected static String indexedShapeType; - protected static String indexedShapePath; - protected static String indexedShapeIndex; - protected static String indexedShapeRouting; - protected static ShapeBuilder indexedShapeToReturn; - - @Override - protected boolean enableWarningsCheck() { - return false; - } - - protected String fieldName() { - return GEO_SHAPE_FIELD_NAME; - } + private static String indexedShapeId; + private static String indexedShapeType; + private static String indexedShapePath; + private static String indexedShapeIndex; + private static String indexedShapeRouting; + private static ShapeBuilder indexedShapeToReturn; @Override protected GeoShapeQueryBuilder doCreateTestQueryBuilder() { return doCreateTestQueryBuilder(randomBoolean()); } - - protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - // LatLonShape does not support MultiPoint queries - RandomShapeGenerator.ShapeType shapeType = - randomFrom(ShapeType.POINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON); + private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + ShapeType shapeType = ShapeType.randomType(random()); ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { - builder = new GeoShapeQueryBuilder(fieldName(), shape); + builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); } else { indexedShapeToReturn = shape; indexedShapeId = randomAlphaOfLengthBetween(3, 20); indexedShapeType = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); + builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeId, indexedShapeType); if (randomBoolean()) { indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); builder.indexedShapeIndex(indexedShapeIndex); @@ -101,11 +91,15 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { } } if (randomBoolean()) { - if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { - builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS)); - } else { - // LatLonShape does not support CONTAINS: - builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN)); + SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); + // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so + // we try to avoid that combination + while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { + strategy = randomFrom(SpatialStrategy.values()); + } + builder.strategy(strategy); + if (strategy != SpatialStrategy.TERM) { + builder.relation(randomFrom(ShapeRelation.values())); } } @@ -167,28 +161,41 @@ public void testNoFieldName() throws Exception { } public void testNoShape() throws IOException { - expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(fieldName(), null)); + expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null)); } public void testNoIndexedShape() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(fieldName(), null, "type")); + () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null, "type")); assertEquals("either shapeBytes or indexedShapeId and indexedShapeType are required", e.getMessage()); } public void testNoIndexedShapeType() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(fieldName(), "id", null)); + () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, "id", null)); assertEquals("indexedShapeType is required if indexedShapeId is specified", e.getMessage()); } public void testNoRelation() throws IOException { ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(fieldName(), shape); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.relation(null)); assertEquals("No Shape Relation defined", e.getMessage()); } + public void testInvalidRelation() throws IOException { + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); + expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); + GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder3.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + } + // see #3878 public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(0, 0), new Coordinate(10, 10)); @@ -198,7 +205,7 @@ public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { public void testFromJson() throws IOException { String json = - "{\n" + + "{\n" + " \"geo_shape\" : {\n" + " \"location\" : {\n" + " \"shape\" : {\n" + @@ -223,7 +230,7 @@ public void testMustRewrite() throws IOException { UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> query.toQuery(createShardContext())); assertEquals("query must be rewritten first", e.getMessage()); QueryBuilder rewrite = rewriteAndFetch(query, createShardContext()); - GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); + GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); geoShapeQueryBuilder.strategy(query.strategy()); geoShapeQueryBuilder.relation(query.relation()); assertEquals(geoShapeQueryBuilder, rewrite); @@ -237,7 +244,7 @@ public void testMultipleRewrite() throws IOException { builder = rewriteAndFetch(builder, createShardContext()); - GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); + GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); expectedShape.strategy(shape.strategy()); expectedShape.relation(shape.relation()); QueryBuilder expected = new BoolQueryBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java deleted file mode 100644 index 3cf6f2031810a..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.SpatialStrategy; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.test.geo.RandomShapeGenerator; -import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; - -import java.io.IOException; - -public class LegacyGeoShapeFieldQueryTests extends GeoShapeQueryBuilderTests { - - @Override - protected String fieldName() { - return LEGACY_GEO_SHAPE_FIELD_NAME; - } - - @Override - protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - ShapeType shapeType = ShapeType.randomType(random()); - ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); - GeoShapeQueryBuilder builder; - clearShapeFields(); - if (indexedShape == false) { - builder = new GeoShapeQueryBuilder(fieldName(), shape); - } else { - indexedShapeToReturn = shape; - indexedShapeId = randomAlphaOfLengthBetween(3, 20); - indexedShapeType = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); - if (randomBoolean()) { - indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapeIndex(indexedShapeIndex); - } - if (randomBoolean()) { - indexedShapePath = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapePath(indexedShapePath); - } - if (randomBoolean()) { - indexedShapeRouting = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapeRouting(indexedShapeRouting); - } - } - if (randomBoolean()) { - SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); - // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so - // we try to avoid that combination - while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { - strategy = randomFrom(SpatialStrategy.values()); - } - builder.strategy(strategy); - if (strategy != SpatialStrategy.TERM) { - builder.relation(randomFrom(ShapeRelation.values())); - } - } - - if (randomBoolean()) { - builder.ignoreUnmapped(randomBoolean()); - } - return builder; - } - - public void testInvalidRelation() throws IOException { - ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); - expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); - GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder3.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index 184ee2759c15e..1067ed62db46e 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -62,7 +62,6 @@ import static org.hamcrest.Matchers.notNullValue; public class MatchQueryBuilderTests extends AbstractQueryTestCase { - @Override protected MatchQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME, diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 1c34057457a4c..70f504516ec8a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1048,12 +1048,6 @@ public void testDisabledFieldNamesField() throws Exception { "_field_names", "enabled=true"))), MapperService.MergeReason.MAPPING_UPDATE); } - assertWarnings(new String[] { - "Field parameter [tree_levels] is deprecated and will be removed in a future version.", - "Field parameter [precision] is deprecated and will be removed in a future version.", - "Field parameter [strategy] is deprecated and will be removed in a future version.", - "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." - }); } diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index b26a7ff510a3d..7231dc7f9a9f2 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -380,7 +380,6 @@ public void testBulk() throws Exception { .endObject() .startObject("location") .field("type", "geo_shape") - .field("ignore_malformed", true) .endObject() .endObject() .endObject() diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index b120b54687607..854872730e8fd 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -45,21 +45,21 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { public void testOrientationPersistence() throws Exception { String idxName = "orientation"; String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); // create index assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); ensureGreen(idxName, idxName+"2"); @@ -144,8 +144,9 @@ public void testIndexShapeRouting() throws Exception { String source = "{\n" + " \"shape\" : {\n" + - " \"type\" : \"bbox\",\n" + - " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + + " \"type\" : \"circle\",\n" + + " \"coordinates\" : [-45.0, 45.0],\n" + + " \"radius\" : \"100m\"\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index a64f98df5a6eb..6c90dcf59e952 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -19,21 +19,16 @@ package org.elasticsearch.search.geo; -import com.carrotsearch.randomizedtesting.generators.RandomNumbers; -import org.apache.lucene.geo.GeoTestUtil; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.LineStringBuilder; -import org.elasticsearch.common.geo.builders.MultiPointBuilder; -import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.settings.Settings; @@ -41,9 +36,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -70,26 +63,12 @@ import static org.hamcrest.Matchers.nullValue; public class GeoShapeQueryTests extends ESSingleNodeTestCase { - private static final String[] PREFIX_TREES = new String[] { - LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.GEOHASH, - LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.QUADTREE - }; - - private XContentBuilder createMapping() throws Exception { - XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape"); - if (randomBoolean()) { - xcb = xcb.field("tree", randomFrom(PREFIX_TREES)) - .field("strategy", randomFrom(SpatialStrategy.RECURSIVE, SpatialStrategy.TERM)); - } - xcb = xcb.endObject().endObject().endObject().endObject(); - - return xcb; - } - public void testNullShape() throws Exception { - String mapping = Strings.toString(createMapping()); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -100,7 +79,12 @@ public void testNullShape() throws Exception { } public void testIndexPointsFilterRectangle() throws Exception { - String mapping = Strings.toString(createMapping()); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -142,11 +126,12 @@ public void testIndexPointsFilterRectangle() throws Exception { } public void testEdgeCases() throws Exception { - XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject().endObject().endObject(); - String mapping = Strings.toString(xcb); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); ensureGreen(); @@ -178,7 +163,12 @@ public void testEdgeCases() throws Exception { } public void testIndexedShapeReference() throws Exception { - String mapping = Strings.toString(createMapping()); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).get(); createIndex("shapes"); ensureGreen(); @@ -215,7 +205,14 @@ public void testIndexedShapeReference() throws Exception { } public void testIndexedShapeReferenceSourceDisabled() throws Exception { - XContentBuilder mapping = createMapping(); + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject() + .endObject() + .endObject(); client().admin().indices().prepareCreate("test").addMapping("type1", mapping).get(); createIndex("shapes", Settings.EMPTY, "shape_type", "_source", "enabled=false"); ensureGreen(); @@ -329,107 +326,24 @@ public void testShapeFetchingPath() throws Exception { assertHitCount(result, 1); } - public void testQueryRandomGeoCollection() throws Exception { + public void testShapeFilterWithRandomGeoCollection() throws Exception { // Create a random geometry collection. GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); - org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); - CoordinatesBuilder cb = new CoordinatesBuilder(); - for (int i = 0; i < randomPoly.numPoints(); ++i) { - cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); - } - gcb.shape(new PolygonBuilder(cb)); - - logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - - if (randomBoolean()) { - client().admin().indices().prepareCreate("test") - .addMapping("type", "location", "type=geo_shape").get(); - } else { - client().admin().indices().prepareCreate("test") - .addMapping("type", "location", "type=geo_shape,tree=quadtree").get(); - } - - XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); - client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); - - ShapeBuilder filterShape = (gcb.getShapeAt(gcb.numShapes() - 1)); - - GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", filterShape); - geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); - assertSearchResponse(result); - assertHitCount(result, 1); - } - - public void testRandomGeoCollectionQuery() throws Exception { - boolean usePrefixTrees = randomBoolean(); - // Create a random geometry collection to index. - GeometryCollectionBuilder gcb; - if (usePrefixTrees) { - gcb = RandomShapeGenerator.createGeometryCollection(random()); - } else { - // vector strategy does not yet support multipoint queries - gcb = new GeometryCollectionBuilder(); - int numShapes = RandomNumbers.randomIntBetween(random(), 1, 4); - for (int i = 0; i < numShapes; ++i) { - ShapeBuilder shape; - do { - shape = RandomShapeGenerator.createShape(random()); - } while (shape instanceof MultiPointBuilder); - gcb.shape(shape); - } - } - org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); - CoordinatesBuilder cb = new CoordinatesBuilder(); - for (int i = 0; i < randomPoly.numPoints(); ++i) { - cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); - } - gcb.shape(new PolygonBuilder(cb)); logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - if (usePrefixTrees == false) { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") - .execute().actionGet(); - } else { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") - .execute().actionGet(); - } + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .get(); XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); - // Create a random geometry collection to query - GeometryCollectionBuilder queryCollection = RandomShapeGenerator.createGeometryCollection(random()); - queryCollection.shape(new PolygonBuilder(cb)); - - GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", queryCollection); - geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); - assertSearchResponse(result); - assertTrue(result.getHits().getTotalHits().value > 0); - } - - /** tests querying a random geometry collection with a point */ - public void testPointQuery() throws Exception { - // Create a random geometry collection to index. - GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); - double[] pt = new double[] {GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()}; - PointBuilder pb = new PointBuilder(pt[0], pt[1]); - gcb.shape(pb); - if (randomBoolean()) { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") - .execute().actionGet(); - } else { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") - .execute().actionGet(); - } - XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); - client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1))); - GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", pb); - geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); + GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape); + filter.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) + .setPostFilter(filter).get(); assertSearchResponse(result); assertHitCount(result, 1); } @@ -461,28 +375,6 @@ public void testContainsShapeQuery() throws Exception { assertThat(response.getHits().getTotalHits().value, greaterThan(0L)); } - public void testExistsQuery() throws Exception { - // Create a random geometry collection. - GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); - logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - - if (randomBoolean()) { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") - .execute().actionGet(); - } else { - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") - .execute().actionGet(); - } - - XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); - client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); - - ExistsQueryBuilder eqb = QueryBuilders.existsQuery("location"); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(eqb).get(); - assertSearchResponse(result); - assertHitCount(result, 1); - } - public void testShapeFilterWithDefinedGeoCollection() throws Exception { createIndex("shapes"); client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") diff --git a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java deleted file mode 100644 index 574bdd46bba5b..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.geo; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.test.ESIntegTestCase; - -import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - -public class LegacyGeoShapeIntegrationIT extends ESIntegTestCase { - - /** - * Test that orientation parameter correctly persists across cluster restart - */ - public void testOrientationPersistence() throws Exception { - String idxName = "orientation"; - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); - - // create index - assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); - - mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); - - assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); - ensureGreen(idxName, idxName+"2"); - - internalCluster().fullRestart(); - ensureGreen(idxName, idxName+"2"); - - // left orientation test - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName)); - IndexService indexService = indicesService.indexService(resolveIndex(idxName)); - MappedFieldType fieldType = indexService.mapperService().fullName("location"); - assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); - - LegacyGeoShapeFieldMapper.GeoShapeFieldType gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; - ShapeBuilder.Orientation orientation = gsfm.orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); - - // right orientation test - indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2")); - indexService = indicesService.indexService(resolveIndex((idxName+"2"))); - fieldType = indexService.mapperService().fullName("location"); - assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); - - gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; - orientation = gsfm.orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); - } - - /** - * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document - */ - public void testIgnoreMalformed() throws Exception { - // create index - assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("geometry", "shape", "type=geo_shape,tree=quadtree,ignore_malformed=true").get()); - ensureGreen(); - - // test self crossing ccw poly not crossing dateline - String polygonGeoJson = Strings.toString(XContentFactory.jsonBuilder().startObject().field("type", "Polygon") - .startArray("coordinates") - .startArray() - .startArray().value(176.0).value(15.0).endArray() - .startArray().value(-177.0).value(10.0).endArray() - .startArray().value(-177.0).value(-10.0).endArray() - .startArray().value(176.0).value(-15.0).endArray() - .startArray().value(-177.0).value(15.0).endArray() - .startArray().value(172.0).value(0.0).endArray() - .startArray().value(176.0).value(15.0).endArray() - .endArray() - .endArray() - .endObject()); - - indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape", - polygonGeoJson)); - SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get(); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); - } - - /** - * Test that the indexed shape routing can be provided if it is required - */ - public void testIndexShapeRouting() throws Exception { - String mapping = "{\n" + - " \"_routing\": {\n" + - " \"required\": true\n" + - " },\n" + - " \"properties\": {\n" + - " \"shape\": {\n" + - " \"type\": \"geo_shape\",\n" + - " \"tree\" : \"quadtree\"\n" + - " }\n" + - " }\n" + - " }"; - - - // create index - assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping, XContentType.JSON).get()); - ensureGreen(); - - String source = "{\n" + - " \"shape\" : {\n" + - " \"type\" : \"bbox\",\n" + - " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + - " }\n" + - "}"; - - indexRandom(true, client().prepareIndex("test", "doc", "0").setSource(source, XContentType.JSON).setRouting("ABC")); - - SearchResponse searchResponse = client().prepareSearch("test").setQuery( - geoShapeQuery("shape", "0", "doc").indexedShapeIndex("test").indexedShapeRouting("ABC") - ).get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); - } - - private String findNodeName(String index) { - ClusterState state = client().admin().cluster().prepareState().get().getState(); - IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); - String nodeId = shard.assignedShards().get(0).currentNodeId(); - return state.getNodes().get(nodeId).getName(); - } -} diff --git a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 0d964e8eb6fa7..76d18a59f9f63 100644 --- a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.search.geo.GeoShapeQueryTests; import org.junit.Assert; import org.locationtech.spatial4j.context.jts.JtsSpatialContext; import org.locationtech.spatial4j.distance.DistanceUtils; @@ -154,7 +153,6 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit /** * Creates a random shape useful for randomized testing, NOTE: exercise caution when using this to build random GeometryCollections * as creating a large random number of random shapes can result in massive resource consumption - * see: {@link GeoShapeQueryTests#testQueryRandomGeoCollection()} * * The following options are included * @param nearPoint Create a shape near a provided point diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index daf29e46b0519..5eef0a249b687 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -113,7 +113,6 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; - protected static final String LEGACY_GEO_SHAPE_FIELD_NAME = "mapped_legacy_geo_shape"; protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, @@ -218,28 +217,12 @@ public void beforeTest() throws Exception { AbstractBuilderTestCase.this, false); return null; }); - if (enableWarningsCheck() == true) { - assertDeprecatedGeoWarnings(); - } } serviceHolder.clientInvocationHandler.delegate = this; serviceHolderWithNoType.clientInvocationHandler.delegate = this; } - protected void assertDeprecatedGeoWarnings() { - String prefix = "Field parameter ["; - String postfix = "] is deprecated and will be removed in a future version."; - String[] deprecationWarnings = new String[] { - prefix + "tree" + postfix, - prefix + "tree_levels" + postfix, - prefix + "precision" + postfix, - prefix + "strategy" + postfix, - prefix + "distance_error_pct" + postfix - }; - assertWarnings(deprecationWarnings); - } - protected static SearchContext getSearchContext(QueryShardContext context) { TestSearchContext testSearchContext = new TestSearchContext(context) { @Override @@ -413,8 +396,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { OBJECT_FIELD_NAME, "type=object", GEO_POINT_FIELD_NAME, "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, "type=geo_shape", - LEGACY_GEO_SHAPE_FIELD_NAME, "type=geo_shape,tree=quadtree" + GEO_SHAPE_FIELD_NAME, "type=geo_shape" ))), MapperService.MergeReason.MAPPING_UPDATE); // also add mappings for two inner field in the object field mapperService.merge("_doc", new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\"," From 16ba5bf764ab24c1d926e039c362ac342c8d23cf Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Tue, 18 Dec 2018 10:05:50 +0200 Subject: [PATCH 51/57] Enhance Invalidate Token API (#35388) This change: - Adds functionality to invalidate all (refresh+access) tokens for all users of a realm - Adds functionality to invalidate all (refresh+access)tokens for a user in all realms - Adds functionality to invalidate all (refresh+access) tokens for a user in a specific realm - Changes the response format for the invalidate token API to contain information about the number of the invalidated tokens and possible errors that were encountered. - Updates the API Documentation After back-porting to 6.x, the `created` field will be removed from master as a field in the response Resolves: #35115 Relates: #34556 --- .../SecurityDocumentationIT.java | 1 + .../security/invalidate-token.asciidoc | 2 +- .../security/invalidate-tokens.asciidoc | 90 +++- .../action/token/InvalidateTokenAction.java | 2 +- .../action/token/InvalidateTokenRequest.java | 133 ++++- .../token/InvalidateTokenRequestBuilder.java | 16 + .../action/token/InvalidateTokenResponse.java | 66 ++- .../support/TokensInvalidationResult.java | 113 ++++ .../core/security/client/SecurityClient.java | 4 + .../action/token/CreateTokenRequestTests.java | 1 - .../token/InvalidateTokenRequestTests.java | 82 +++ .../token/InvalidateTokenResponseTests.java | 141 +++++ .../TransportSamlInvalidateSessionAction.java | 46 +- .../saml/TransportSamlLogoutAction.java | 3 +- .../token/TransportInvalidateTokenAction.java | 11 +- .../xpack/security/authc/TokenService.java | 502 ++++++++++++------ .../oauth2/RestInvalidateTokenAction.java | 66 +-- ...sportSamlInvalidateSessionActionTests.java | 72 ++- .../saml/TransportSamlLogoutActionTests.java | 49 +- .../security/authc/TokenAuthIntegTests.java | 96 +++- .../security/authc/TokenServiceTests.java | 3 +- .../TokensInvalidationResultTests.java | 74 +++ .../RestInvalidateTokenActionTests.java | 61 +++ .../rest-api-spec/test/token/10_basic.yml | 90 +++- 24 files changed, 1422 insertions(+), 302 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/TokensInvalidationResultTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenActionTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index 8bd285cd31f51..6cd56774086a1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -1317,6 +1317,7 @@ public void onFailure(Exception e) { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/36362") public void testInvalidateToken() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/docs/java-rest/high-level/security/invalidate-token.asciidoc b/docs/java-rest/high-level/security/invalidate-token.asciidoc index ecb3fedb56f0a..65e0f15bd86fc 100644 --- a/docs/java-rest/high-level/security/invalidate-token.asciidoc +++ b/docs/java-rest/high-level/security/invalidate-token.asciidoc @@ -36,4 +36,4 @@ The returned +{response}+ contains a single property: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-response] --------------------------------------------------- +-------------------------------------------------- \ No newline at end of file diff --git a/x-pack/docs/en/rest-api/security/invalidate-tokens.asciidoc b/x-pack/docs/en/rest-api/security/invalidate-tokens.asciidoc index 540f586682595..18c88f7addd62 100644 --- a/x-pack/docs/en/rest-api/security/invalidate-tokens.asciidoc +++ b/x-pack/docs/en/rest-api/security/invalidate-tokens.asciidoc @@ -2,7 +2,7 @@ [[security-api-invalidate-token]] === Invalidate token API -Invalidates an access token or a refresh token. +Invalidates one or more access tokens or refresh tokens. ==== Request @@ -19,21 +19,31 @@ can no longer be used. That time period is defined by the The refresh tokens returned by the <> are only valid for 24 hours. They can also be used exactly once. -If you want to invalidate an access or refresh token immediately, use this invalidate token API. +If you want to invalidate one or more access or refresh tokens immediately, use this invalidate token API. ==== Request Body The following parameters can be specified in the body of a DELETE request and -pertain to invalidating a token: +pertain to invalidating tokens: `token` (optional):: -(string) An access token. This parameter cannot be used when `refresh_token` is used. +(string) An access token. This parameter cannot be used any of `refresh_token`, `realm_name` or + `username` are used. `refresh_token` (optional):: -(string) A refresh token. This parameter cannot be used when `token` is used. +(string) A refresh token. This parameter cannot be used any of `refresh_token`, `realm_name` or + `username` are used. -NOTE: One of `token` or `refresh_token` parameters is required. +`realm_name` (optional):: +(string) The name of an authentication realm. This parameter cannot be used with either `refresh_token` or `token`. + +`username` (optional):: +(string) The username of a user. This parameter cannot be used with either `refresh_token` or `token` + +NOTE: While all parameters are optional, at least one of them is required. More specifically, either one of `token` +or `refresh_token` parameters is required. If none of these two are specified, then `realm_name` and/or `username` +need to be specified. ==== Examples @@ -59,15 +69,75 @@ DELETE /_security/oauth2/token -------------------------------------------------- // NOTCONSOLE -A successful call returns a JSON structure that indicates whether the token -has already been invalidated. +The following example invalidates all access tokens and refresh tokens for the `saml1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/oauth2/token +{ + "realm_name" : "saml1" +} +-------------------------------------------------- +// NOTCONSOLE + +The following example invalidates all access tokens and refresh tokens for the user `myuser` in all realms immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/oauth2/token +{ + "username" : "myuser" +} +-------------------------------------------------- +// NOTCONSOLE + +Finally, the following example invalidates all access tokens and refresh tokens for the user `myuser` in + the `saml1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/security/oauth2/token +{ + "username" : "myuser", + "realm_name" : "saml1" +} +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that contains the number of tokens that were invalidated, the number +of tokens that had already been invalidated, and potentially a list of errors encountered while invalidating +specific tokens. [source,js] -------------------------------------------------- { - "created" : true <1> + "invalidated_tokens":9, <1> + "previously_invalidated_tokens":15, <2> + "error_count":2, <3> + "error_details":[ <4> + { + "type":"exception", + "reason":"Elasticsearch exception [type=exception, reason=foo]", + "caused_by":{ + "type":"exception", + "reason":"Elasticsearch exception [type=illegal_argument_exception, reason=bar]" + } + }, + { + "type":"exception", + "reason":"Elasticsearch exception [type=exception, reason=boo]", + "caused_by":{ + "type":"exception", + "reason":"Elasticsearch exception [type=illegal_argument_exception, reason=far]" + } + } + ] } -------------------------------------------------- // NOTCONSOLE -<1> When a token has already been invalidated, `created` is set to false. +<1> The number of the tokens that were invalidated as part of this request. +<2> The number of tokens that were already invalidated. +<3> The number of errors that were encountered when invalidating the tokens. +<4> Details about these errors. This field is not present in the response when + `error_count` is 0. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java index 679ee0756f638..57bd5bd35dd0c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java @@ -8,7 +8,7 @@ import org.elasticsearch.action.Action; /** - * Action for invalidating a given token + * Action for invalidating one or more tokens */ public final class InvalidateTokenAction extends Action { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequest.java index 7a8372fe456d3..de3b73ec4afce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequest.java @@ -8,6 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -22,31 +23,81 @@ public final class InvalidateTokenRequest extends ActionRequest { public enum Type { - ACCESS_TOKEN, - REFRESH_TOKEN + ACCESS_TOKEN("token"), + REFRESH_TOKEN("refresh_token"); + + private final String value; + + Type(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static Type fromString(String tokenType) { + if (tokenType != null) { + for (Type type : values()) { + if (type.getValue().equals(tokenType)) { + return type; + } + } + } + return null; + } } private String tokenString; private Type tokenType; + private String realmName; + private String userName; public InvalidateTokenRequest() {} /** - * @param tokenString the string representation of the token + * @param tokenString the string representation of the token to be invalidated + * @param tokenType the type of the token to be invalidated + * @param realmName the name of the realm for which all tokens will be invalidated + * @param userName the principal of the user for which all tokens will be invalidated */ - public InvalidateTokenRequest(String tokenString, Type type) { + public InvalidateTokenRequest(@Nullable String tokenString, @Nullable String tokenType, + @Nullable String realmName, @Nullable String userName) { this.tokenString = tokenString; - this.tokenType = type; + this.tokenType = Type.fromString(tokenType); + this.realmName = realmName; + this.userName = userName; + } + + /** + * @param tokenString the string representation of the token to be invalidated + * @param tokenType the type of the token to be invalidated + */ + public InvalidateTokenRequest(String tokenString, String tokenType) { + this.tokenString = tokenString; + this.tokenType = Type.fromString(tokenType); + this.realmName = null; + this.userName = null; } @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (Strings.isNullOrEmpty(tokenString)) { - validationException = addValidationError("token string must be provided", null); - } - if (tokenType == null) { - validationException = addValidationError("token type must be provided", validationException); + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + if (Strings.hasText(tokenString)) { + validationException = + addValidationError("token string must not be provided when realm name or username is specified", null); + } + if (tokenType != null) { + validationException = + addValidationError("token type must not be provided when realm name or username is specified", validationException); + } + } else if (Strings.isNullOrEmpty(tokenString)) { + validationException = + addValidationError("token string must be provided when not specifying a realm name or a username", null); + } else if (tokenType == null) { + validationException = + addValidationError("token type must be provided when a token string is specified", null); } return validationException; } @@ -67,26 +118,76 @@ void setTokenType(Type tokenType) { this.tokenType = tokenType; } + public String getRealmName() { + return realmName; + } + + public void setRealmName(String realmName) { + this.realmName = realmName; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(tokenString); + if (out.getVersion().before(Version.V_7_0_0)) { + if (Strings.isNullOrEmpty(tokenString)) { + throw new IllegalArgumentException("token is required for versions < v6.6.0"); + } + out.writeString(tokenString); + } else { + out.writeOptionalString(tokenString); + } if (out.getVersion().onOrAfter(Version.V_6_2_0)) { - out.writeVInt(tokenType.ordinal()); + if (out.getVersion().before(Version.V_7_0_0)) { + if (tokenType == null) { + throw new IllegalArgumentException("token type is not optional for versions > v6.2.0 and < v6.6.0"); + } + out.writeVInt(tokenType.ordinal()); + } else { + out.writeOptionalVInt(tokenType == null ? null : tokenType.ordinal()); + } } else if (tokenType == Type.REFRESH_TOKEN) { - throw new IllegalArgumentException("refresh token invalidation cannot be serialized with version [" + out.getVersion() + - "]"); + throw new IllegalArgumentException("refresh token invalidation cannot be serialized with version [" + out.getVersion() + "]"); + } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalString(realmName); + out.writeOptionalString(userName); + } else if (realmName != null || userName != null) { + throw new IllegalArgumentException( + "realm or user token invalidation cannot be serialized with version [" + out.getVersion() + "]"); } } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - tokenString = in.readString(); + if (in.getVersion().before(Version.V_7_0_0)) { + tokenString = in.readString(); + } else { + tokenString = in.readOptionalString(); + } if (in.getVersion().onOrAfter(Version.V_6_2_0)) { - tokenType = Type.values()[in.readVInt()]; + if (in.getVersion().before(Version.V_7_0_0)) { + int type = in.readVInt(); + tokenType = Type.values()[type]; + } else { + Integer type = in.readOptionalVInt(); + tokenType = type == null ? null : Type.values()[type]; + } } else { tokenType = Type.ACCESS_TOKEN; } + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestBuilder.java index f77f6c6533271..0b454905cfa54 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestBuilder.java @@ -34,4 +34,20 @@ public InvalidateTokenRequestBuilder setType(InvalidateTokenRequest.Type type) { request.setTokenType(type); return this; } + + /** + * Sets the name of the realm for which all tokens should be invalidated + */ + public InvalidateTokenRequestBuilder setRealmName(String realmName) { + request.setRealmName(realmName); + return this; + } + + /** + * Sets the username for which all tokens should be invalidated + */ + public InvalidateTokenRequestBuilder setUserName(String username) { + request.setUserName(username); + return this; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java index cebb005b27254..886caeac370fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java @@ -5,41 +5,83 @@ */ package org.elasticsearch.xpack.core.security.action.token; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; /** - * Response for a invalidation of a token. + * Response for a invalidation of one or multiple tokens. */ -public final class InvalidateTokenResponse extends ActionResponse { +public final class InvalidateTokenResponse extends ActionResponse implements ToXContent { - private boolean created; + private TokensInvalidationResult result; public InvalidateTokenResponse() {} - public InvalidateTokenResponse(boolean created) { - this.created = created; + public InvalidateTokenResponse(TokensInvalidationResult result) { + this.result = result; } - /** - * If the token is already invalidated then created will be false - */ - public boolean isCreated() { - return created; + public TokensInvalidationResult getResult() { + return result; + } + + private boolean isCreated() { + return result.getInvalidatedTokens().size() > 0 + && result.getPreviouslyInvalidatedTokens().isEmpty() + && result.getErrors().isEmpty(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeBoolean(created); + if (out.getVersion().before(Version.V_7_0_0)) { + out.writeBoolean(isCreated()); + } else { + result.writeTo(out); + } } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - created = in.readBoolean(); + if (in.getVersion().before(Version.V_7_0_0)) { + final boolean created = in.readBoolean(); + if (created) { + result = new TokensInvalidationResult(Arrays.asList(""), Collections.emptyList(), Collections.emptyList(), 0); + } else { + result = new TokensInvalidationResult(Collections.emptyList(), Arrays.asList(""), Collections.emptyList(), 0); + } + } else { + result = new TokensInvalidationResult(in); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + result.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InvalidateTokenResponse that = (InvalidateTokenResponse) o; + return Objects.equals(result, that.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java new file mode 100644 index 0000000000000..cfa83b63ed54f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authc.support; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The result of attempting to invalidate one or multiple tokens. The result contains information about: + *

      + *
    • how many of the tokens were actually invalidated
      • + *
    • how many tokens are not invalidated in this request because they were already invalidated
      • + *
    • how many errors were encountered while invalidating tokens and the error details
      • + *
    + */ +public class TokensInvalidationResult implements ToXContentObject, Writeable { + + private final List invalidatedTokens; + private final List previouslyInvalidatedTokens; + private final List errors; + private final int attemptCount; + + public TokensInvalidationResult(List invalidatedTokens, List previouslyInvalidatedTokens, + @Nullable List errors, int attemptCount) { + Objects.requireNonNull(invalidatedTokens, "invalidated_tokens must be provided"); + this.invalidatedTokens = invalidatedTokens; + Objects.requireNonNull(previouslyInvalidatedTokens, "previously_invalidated_tokens must be provided"); + this.previouslyInvalidatedTokens = previouslyInvalidatedTokens; + if (null != errors) { + this.errors = errors; + } else { + this.errors = Collections.emptyList(); + } + this.attemptCount = attemptCount; + } + + public TokensInvalidationResult(StreamInput in) throws IOException { + this.invalidatedTokens = in.readList(StreamInput::readString); + this.previouslyInvalidatedTokens = in.readList(StreamInput::readString); + this.errors = in.readList(StreamInput::readException); + this.attemptCount = in.readVInt(); + } + + public static TokensInvalidationResult emptyResult() { + return new TokensInvalidationResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), 0); + } + + + public List getInvalidatedTokens() { + return invalidatedTokens; + } + + public List getPreviouslyInvalidatedTokens() { + return previouslyInvalidatedTokens; + } + + public List getErrors() { + return errors; + } + + public int getAttemptCount() { + return attemptCount; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + //Remove created after PR is backported to 6.x + .field("created", isCreated()) + .field("invalidated_tokens", invalidatedTokens.size()) + .field("previously_invalidated_tokens", previouslyInvalidatedTokens.size()) + .field("error_count", errors.size()); + if (errors.isEmpty() == false) { + builder.field("error_details"); + builder.startArray(); + for (ElasticsearchException e : errors) { + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, params, e); + builder.endObject(); + } + builder.endArray(); + } + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringList(invalidatedTokens); + out.writeStringList(previouslyInvalidatedTokens); + out.writeCollection(errors, StreamOutput::writeException); + out.writeVInt(attemptCount); + } + + private boolean isCreated() { + return this.getInvalidatedTokens().size() > 0 + && this.getPreviouslyInvalidatedTokens().isEmpty() + && this.getErrors().isEmpty(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java index ef59f870c6854..a7faf4d223108 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java @@ -326,6 +326,10 @@ public InvalidateTokenRequestBuilder prepareInvalidateToken(String token) { return new InvalidateTokenRequestBuilder(client).setTokenString(token); } + public InvalidateTokenRequestBuilder prepareInvalidateToken() { + return new InvalidateTokenRequestBuilder(client); + } + public void invalidateToken(InvalidateTokenRequest request, ActionListener listener) { client.execute(InvalidateTokenAction.INSTANCE, request, listener); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java index bd23198e8eae7..2d8782f0111e6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasItem; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestTests.java new file mode 100644 index 0000000000000..3fd7eb7da46f7 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenRequestTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.security.action.token; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; + +public class InvalidateTokenRequestTests extends ESTestCase { + + public void testValidation() { + InvalidateTokenRequest request = new InvalidateTokenRequest(); + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), containsString("token string must be provided when not specifying a realm")); + + request = new InvalidateTokenRequest(randomAlphaOfLength(12), randomFrom("", null)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), containsString("token type must be provided when a token string is specified")); + + request = new InvalidateTokenRequest(randomFrom("", null), "access_token"); + ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), containsString("token string must be provided when not specifying a realm")); + + request = new InvalidateTokenRequest(randomFrom("", null), randomFrom("", null), randomAlphaOfLength(4), randomAlphaOfLength(8)); + ve = request.validate(); + assertNull(ve); + + request = + new InvalidateTokenRequest(randomAlphaOfLength(4), randomFrom("", null), randomAlphaOfLength(4), randomAlphaOfLength(8)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), + containsString("token string must not be provided when realm name or username is specified")); + + request = new InvalidateTokenRequest(randomAlphaOfLength(4), randomFrom("token", "refresh_token"), + randomAlphaOfLength(4), randomAlphaOfLength(8)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(2, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), + containsString("token string must not be provided when realm name or username is specified")); + assertThat(ve.validationErrors().get(1), + containsString("token type must not be provided when realm name or username is specified")); + + request = + new InvalidateTokenRequest(randomAlphaOfLength(4), randomFrom("", null), randomAlphaOfLength(4), randomAlphaOfLength(8)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), + containsString("token string must not be provided when realm name or username is specified")); + + request = + new InvalidateTokenRequest(randomAlphaOfLength(4), randomFrom("token", "refresh_token"), randomFrom("", null), + randomAlphaOfLength(8)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(2, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), + containsString("token string must not be provided when realm name or username is specified")); + assertThat(ve.validationErrors().get(1), + containsString("token type must not be provided when realm name or username is specified")); + + request = new InvalidateTokenRequest(randomAlphaOfLength(4), randomFrom("", null), randomFrom("", null), randomAlphaOfLength(8)); + ve = request.validate(); + assertNotNull(ve); + assertEquals(1, ve.validationErrors().size()); + assertThat(ve.validationErrors().get(0), + containsString("token string must not be provided when realm name or username is specified")); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java new file mode 100644 index 0000000000000..1a59971ff9c60 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.security.action.token; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class InvalidateTokenResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false)), + Arrays.asList(generateRandomStringArray(20, 15, false)), + Arrays.asList(new ElasticsearchException("foo", new IllegalArgumentException("this is an error message")), + new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))), + randomIntBetween(0, 5)); + InvalidateTokenResponse response = new InvalidateTokenResponse(result); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateTokenResponse serialized = new InvalidateTokenResponse(); + serialized.readFrom(input); + assertThat(serialized.getResult().getInvalidatedTokens(), equalTo(response.getResult().getInvalidatedTokens())); + assertThat(serialized.getResult().getPreviouslyInvalidatedTokens(), + equalTo(response.getResult().getPreviouslyInvalidatedTokens())); + assertThat(serialized.getResult().getErrors().size(), equalTo(response.getResult().getErrors().size())); + assertThat(serialized.getResult().getErrors().get(0).toString(), containsString("this is an error message")); + assertThat(serialized.getResult().getErrors().get(1).toString(), containsString("this is an error message2")); + } + } + + result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false)), + Arrays.asList(generateRandomStringArray(20, 15, false)), + Collections.emptyList(), randomIntBetween(0, 5)); + response = new InvalidateTokenResponse(result); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateTokenResponse serialized = new InvalidateTokenResponse(); + serialized.readFrom(input); + assertThat(serialized.getResult().getInvalidatedTokens(), equalTo(response.getResult().getInvalidatedTokens())); + assertThat(serialized.getResult().getPreviouslyInvalidatedTokens(), + equalTo(response.getResult().getPreviouslyInvalidatedTokens())); + assertThat(serialized.getResult().getErrors().size(), equalTo(response.getResult().getErrors().size())); + } + } + } + + public void testSerializationToPre66Version() throws IOException{ + final Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_2_0, Version.V_6_5_1); + TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false, false)), + Arrays.asList(generateRandomStringArray(20, 15, false, false)), + Arrays.asList(new ElasticsearchException("foo", new IllegalArgumentException("this is an error message")), + new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))), + randomIntBetween(0, 5)); + InvalidateTokenResponse response = new InvalidateTokenResponse(result); + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + // False as we have errors and previously invalidated tokens + assertThat(input.readBoolean(), equalTo(false)); + } + } + + result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false, false)), + Arrays.asList(generateRandomStringArray(20, 15, false, false)), + Collections.emptyList(), randomIntBetween(0, 5)); + response = new InvalidateTokenResponse(result); + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + // False as we have previously invalidated tokens + assertThat(input.readBoolean(), equalTo(false)); + } + } + + result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false, false)), + Collections.emptyList(), Collections.emptyList(), randomIntBetween(0, 5)); + response = new InvalidateTokenResponse(result); + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + assertThat(input.readBoolean(), equalTo(true)); + } + } + } + + public void testToXContent() throws IOException { + List invalidatedTokens = Arrays.asList(generateRandomStringArray(20, 15, false)); + List previouslyInvalidatedTokens = Arrays.asList(generateRandomStringArray(20, 15, false)); + TokensInvalidationResult result = new TokensInvalidationResult(invalidatedTokens, previouslyInvalidatedTokens, + Arrays.asList(new ElasticsearchException("foo", new IllegalArgumentException("this is an error message")), + new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))), + randomIntBetween(0, 5)); + InvalidateTokenResponse response = new InvalidateTokenResponse(result); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), + equalTo("{\"created\":false," + + "\"invalidated_tokens\":" + invalidatedTokens.size() + "," + + "\"previously_invalidated_tokens\":" + previouslyInvalidatedTokens.size() + "," + + "\"error_count\":2," + + "\"error_details\":[" + + "{\"type\":\"exception\"," + + "\"reason\":\"foo\"," + + "\"caused_by\":{" + + "\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"this is an error message\"}" + + "}," + + "{\"type\":\"exception\"," + + "\"reason\":\"bar\"," + + "\"caused_by\":" + + "{\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"this is an error message2\"}" + + "}" + + "]" + + "}")); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java index f0e6bf2c99063..8c35df01ed94a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionAction; import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionRequest; import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionResponse; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.UserToken; @@ -27,12 +28,11 @@ import org.elasticsearch.xpack.security.authc.saml.SamlUtils; import org.opensaml.saml.saml2.core.LogoutResponse; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; +import java.util.function.Predicate; import static org.elasticsearch.xpack.security.authc.saml.SamlRealm.findSamlRealms; @@ -85,7 +85,7 @@ private String buildLogoutResponseUrl(SamlRealm realm, SamlLogoutRequestHandler. private void findAndInvalidateTokens(SamlRealm realm, SamlLogoutRequestHandler.Result result, ActionListener listener) { final Map tokenMetadata = realm.createTokenMetadata(result.getNameId(), result.getSession()); - if (Strings.hasText((String) tokenMetadata.get(SamlRealm.TOKEN_METADATA_NAMEID_VALUE)) == false) { + if (Strings.isNullOrEmpty((String) tokenMetadata.get(SamlRealm.TOKEN_METADATA_NAMEID_VALUE))) { // If we don't have a valid name-id to match against, don't do anything logger.debug("Logout request [{}] has no NameID value, so cannot invalidate any sessions", result); listener.onResponse(0); @@ -93,22 +93,21 @@ private void findAndInvalidateTokens(SamlRealm realm, SamlLogoutRequestHandler.R } tokenService.findActiveTokensForRealm(realm.name(), ActionListener.wrap(tokens -> { - List> sessionTokens = filterTokens(tokens, tokenMetadata); - logger.debug("Found [{}] token pairs to invalidate for SAML metadata [{}]", sessionTokens.size(), tokenMetadata); - if (sessionTokens.isEmpty()) { - listener.onResponse(0); - } else { - GroupedActionListener groupedListener = new GroupedActionListener<>( - ActionListener.wrap(collection -> listener.onResponse(collection.size()), listener::onFailure), - sessionTokens.size(), Collections.emptyList() - ); - sessionTokens.forEach(tuple -> invalidateTokenPair(tuple, groupedListener)); - } - }, e -> listener.onFailure(e) - )); + logger.debug("Found [{}] token pairs to invalidate for SAML metadata [{}]", tokens.size(), tokenMetadata); + if (tokens.isEmpty()) { + listener.onResponse(0); + } else { + GroupedActionListener groupedListener = new GroupedActionListener<>( + ActionListener.wrap(collection -> listener.onResponse(collection.size()), listener::onFailure), + tokens.size(), Collections.emptyList() + ); + tokens.forEach(tuple -> invalidateTokenPair(tuple, groupedListener)); + } + }, listener::onFailure + ), containsMetadata(tokenMetadata)); } - private void invalidateTokenPair(Tuple tokenPair, ActionListener listener) { + private void invalidateTokenPair(Tuple tokenPair, ActionListener listener) { // Invalidate the refresh token first, so the client doesn't trigger a refresh once the access token is invalidated tokenService.invalidateRefreshToken(tokenPair.v2(), ActionListener.wrap(ignore -> tokenService.invalidateAccessToken( tokenPair.v1(), @@ -118,13 +117,12 @@ private void invalidateTokenPair(Tuple tokenPair, ActionListe })), listener::onFailure)); } - private List> filterTokens(Collection> tokens, Map requiredMetadata) { - return tokens.stream() - .filter(tup -> { - Map actualMetadata = tup.v1().getMetadata(); - return requiredMetadata.entrySet().stream().allMatch(e -> Objects.equals(actualMetadata.get(e.getKey()), e.getValue())); - }) - .collect(Collectors.toList()); + + private Predicate> containsMetadata(Map requiredMetadata) { + return source -> { + Map actualMetadata = (Map) source.get("metadata"); + return requiredMetadata.entrySet().stream().allMatch(e -> Objects.equals(actualMetadata.get(e.getKey()), e.getValue())); + }; } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java index b62702ead7840..28e9f911cd5e3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Realm; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; @@ -79,7 +80,7 @@ protected void doExecute(Task task, SamlLogoutRequest request, ActionListener listener) { + private void invalidateRefreshToken(String refreshToken, ActionListener listener) { if (refreshToken == null) { listener.onResponse(null); } else { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java index 70f614435fce0..9f0443a86f76a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java @@ -8,12 +8,14 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction; import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequest; import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenResponse; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import org.elasticsearch.xpack.security.authc.TokenService; /** @@ -31,9 +33,12 @@ public TransportInvalidateTokenAction(TransportService transportService, ActionF @Override protected void doExecute(Task task, InvalidateTokenRequest request, ActionListener listener) { - final ActionListener invalidateListener = - ActionListener.wrap(created -> listener.onResponse(new InvalidateTokenResponse(created)), listener::onFailure); - if (request.getTokenType() == InvalidateTokenRequest.Type.ACCESS_TOKEN) { + final ActionListener invalidateListener = + ActionListener.wrap(tokensInvalidationResult -> + listener.onResponse(new InvalidateTokenResponse(tokensInvalidationResult)), listener::onFailure); + if (Strings.hasText(request.getUserName()) || Strings.hasText(request.getRealmName())) { + tokenService.invalidateActiveTokensForRealmAndUser(request.getRealmName(), request.getUserName(), invalidateListener); + } else if (request.getTokenType() == InvalidateTokenRequest.Type.ACCESS_TOKEN) { tokenService.invalidateAccessToken(request.getTokenString(), invalidateListener); } else { assert request.getTokenType() == InvalidateTokenRequest.Type.REFRESH_TOKEN; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index be5b11aa666d1..15d3e75842615 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -17,6 +17,11 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest.OpType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetItemResponse; @@ -24,7 +29,6 @@ import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; @@ -39,6 +43,7 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -61,7 +66,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -74,6 +78,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.KeyAndTimestamp; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import javax.crypto.Cipher; @@ -90,6 +95,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; @@ -116,6 +122,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -221,9 +229,9 @@ public void createUserToken(Authentication authentication, Authentication origin boolean includeRefreshToken) throws IOException { ensureEnabled(); if (authentication == null) { - listener.onFailure(traceLog("create token", null, new IllegalArgumentException("authentication must be provided"))); + listener.onFailure(traceLog("create token", new IllegalArgumentException("authentication must be provided"))); } else if (originatingClientAuth == null) { - listener.onFailure(traceLog("create token", null, + listener.onFailure(traceLog("create token", new IllegalArgumentException("originating client authentication must be provided"))); } else { final Instant created = clock.instant(); @@ -471,7 +479,7 @@ private static void decryptTokenId(StreamInput in, Cipher cipher, Version versio * have been created on versions on or after 6.2; this step involves performing an update to * the token document and setting the invalidated field to true */ - public void invalidateAccessToken(String tokenString, ActionListener listener) { + public void invalidateAccessToken(String tokenString, ActionListener listener) { ensureEnabled(); if (Strings.isNullOrEmpty(tokenString)) { logger.trace("No token-string provided"); @@ -484,7 +492,8 @@ public void invalidateAccessToken(String tokenString, ActionListener li listener.onFailure(traceLog("invalidate token", tokenString, malformedTokenException())); } else { final long expirationEpochMilli = getExpirationTime().toEpochMilli(); - indexBwcInvalidation(userToken, listener, new AtomicInteger(0), expirationEpochMilli); + indexBwcInvalidation(Collections.singleton(userToken.getId()), listener, new AtomicInteger(0), + expirationEpochMilli, null); } }, listener::onFailure)); } catch (IOException e) { @@ -499,7 +508,7 @@ public void invalidateAccessToken(String tokenString, ActionListener li * * @see #invalidateAccessToken(String, ActionListener) */ - public void invalidateAccessToken(UserToken userToken, ActionListener listener) { + public void invalidateAccessToken(UserToken userToken, ActionListener listener) { ensureEnabled(); if (userToken == null) { logger.trace("No access token provided"); @@ -507,11 +516,17 @@ public void invalidateAccessToken(UserToken userToken, ActionListener l } else { maybeStartTokenRemover(); final long expirationEpochMilli = getExpirationTime().toEpochMilli(); - indexBwcInvalidation(userToken, listener, new AtomicInteger(0), expirationEpochMilli); + indexBwcInvalidation(Collections.singleton(userToken.getId()), listener, new AtomicInteger(0), expirationEpochMilli, null); } } - public void invalidateRefreshToken(String refreshToken, ActionListener listener) { + /** + * This method performs the steps necessary to invalidate a refresh token so that it may no longer be used. + * + * @param refreshToken The string representation of the refresh token + * @param listener the listener to notify upon completion + */ + public void invalidateRefreshToken(String refreshToken, ActionListener listener) { ensureEnabled(); if (Strings.isNullOrEmpty(refreshToken)) { logger.trace("No refresh token provided"); @@ -520,152 +535,222 @@ public void invalidateRefreshToken(String refreshToken, ActionListener maybeStartTokenRemover(); findTokenFromRefreshToken(refreshToken, ActionListener.wrap(tuple -> { - final String docId = tuple.v1().getHits().getAt(0).getId(); - final long docVersion = tuple.v1().getHits().getAt(0).getVersion(); - indexInvalidation(docId, Version.CURRENT, listener, tuple.v2(), "refresh_token", docVersion); + final String docId = getTokenIdFromDocumentId(tuple.v1().getHits().getAt(0).getId()); + indexInvalidation(Collections.singletonList(docId), listener, tuple.v2(), "refresh_token", null); }, listener::onFailure), new AtomicInteger(0)); } } /** - * Performs the actual bwc invalidation of a token and then kicks off the new invalidation method + * Invalidate all access tokens and all refresh tokens of a given {@code realmName} and/or of a given + * {@code username} so that they may no longer be used * - * @param userToken the token to invalidate + * @param realmName the realm of which the tokens should be invalidated + * @param username the username for which the tokens should be invalidated + * @param listener the listener to notify upon completion + */ + public void invalidateActiveTokensForRealmAndUser(@Nullable String realmName, @Nullable String username, + ActionListener listener) { + ensureEnabled(); + if (Strings.isNullOrEmpty(realmName) && Strings.isNullOrEmpty(username)) { + logger.trace("No realm name or username provided"); + listener.onFailure(new IllegalArgumentException("realm name or username must be provided")); + } else { + if (Strings.isNullOrEmpty(realmName)) { + findActiveTokensForUser(username, ActionListener.wrap(tokenTuples -> { + if (tokenTuples.isEmpty()) { + logger.warn("No tokens to invalidate for realm [{}] and username [{}]", realmName, username); + listener.onResponse(TokensInvalidationResult.emptyResult()); + } else { + invalidateAllTokens(tokenTuples.stream().map(t -> t.v1().getId()).collect(Collectors.toList()), listener); + } + }, listener::onFailure)); + } else { + Predicate filter = null; + if (Strings.hasText(username)) { + filter = isOfUser(username); + } + findActiveTokensForRealm(realmName, ActionListener.wrap(tokenTuples -> { + if (tokenTuples.isEmpty()) { + logger.warn("No tokens to invalidate for realm [{}] and username [{}]", realmName, username); + listener.onResponse(TokensInvalidationResult.emptyResult()); + } else { + invalidateAllTokens(tokenTuples.stream().map(t -> t.v1().getId()).collect(Collectors.toList()), listener); + } + }, listener::onFailure), filter); + } + } + } + + /** + * Invalidates a collection of access_token and refresh_token that were retrieved by + * {@link TokenService#invalidateActiveTokensForRealmAndUser} + * + * @param accessTokenIds The ids of the access tokens which should be invalidated (along with the respective refresh_token) + * @param listener the listener to notify upon completion + */ + private void invalidateAllTokens(Collection accessTokenIds, ActionListener listener) { + maybeStartTokenRemover(); + final long expirationEpochMilli = getExpirationTime().toEpochMilli(); + // Invalidate the refresh tokens first so that they cannot be used to get new + // access tokens while we invalidate the access tokens we currently know about + indexInvalidation(accessTokenIds, ActionListener.wrap(result -> + indexBwcInvalidation(accessTokenIds, listener, new AtomicInteger(result.getAttemptCount()), + expirationEpochMilli, result), + listener::onFailure), new AtomicInteger(0), "refresh_token", null); + } + + /** + * Performs the actual bwc invalidation of a collection of tokens and then kicks off the new invalidation method. + * + * @param tokenIds the collection of token ids or token document ids that should be invalidated * @param listener the listener to notify upon completion * @param attemptCount the number of attempts to invalidate that have already been tried * @param expirationEpochMilli the expiration time as milliseconds since the epoch + * @param previousResult if this not the initial attempt for invalidation, it contains the result of invalidating + * tokens up to the point of the retry. This result is added to the result of the current attempt */ - private void indexBwcInvalidation(UserToken userToken, ActionListener listener, AtomicInteger attemptCount, - long expirationEpochMilli) { - if (attemptCount.get() > MAX_RETRY_ATTEMPTS) { - logger.warn("Failed to invalidate token [{}] after [{}] attempts", userToken.getId(), attemptCount.get()); - listener.onFailure(invalidGrantException("failed to invalidate token")); + private void indexBwcInvalidation(Collection tokenIds, ActionListener listener, + AtomicInteger attemptCount, long expirationEpochMilli, + @Nullable TokensInvalidationResult previousResult) { + + if (tokenIds.isEmpty()) { + logger.warn("No tokens provided for invalidation"); + listener.onFailure(invalidGrantException("No tokens provided for invalidation")); + } else if (attemptCount.get() > MAX_RETRY_ATTEMPTS) { + logger.warn("Failed to invalidate [{}] tokens after [{}] attempts", tokenIds.size(), + attemptCount.get()); + listener.onFailure(invalidGrantException("failed to invalidate tokens")); } else { - final String invalidatedTokenId = getInvalidatedTokenDocumentId(userToken); - IndexRequest indexRequest = client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, invalidatedTokenId) + BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + for (String tokenId : tokenIds) { + final String invalidatedTokenId = getInvalidatedTokenDocumentId(tokenId); + IndexRequest indexRequest = client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, invalidatedTokenId) .setOpType(OpType.CREATE) .setSource("doc_type", INVALIDATED_TOKEN_DOC_TYPE, "expiration_time", expirationEpochMilli) - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - final String tokenDocId = getTokenDocumentId(userToken); - final Version version = userToken.getVersion(); - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", tokenDocId, ex)), - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, indexRequest, - ActionListener.wrap(indexResponse -> { - ActionListener wrappedListener = - ActionListener.wrap(ignore -> listener.onResponse(true), listener::onFailure); - indexInvalidation(tokenDocId, version, wrappedListener, attemptCount, "access_token", 1L); - }, e -> { - Throwable cause = ExceptionsHelper.unwrapCause(e); - traceLog("(bwc) invalidate token", tokenDocId, cause); - if (cause instanceof VersionConflictEngineException) { - // expected since something else could have invalidated - ActionListener wrappedListener = - ActionListener.wrap(ignore -> listener.onResponse(false), listener::onFailure); - indexInvalidation(tokenDocId, version, wrappedListener, attemptCount, "access_token", 1L); - } else if (isShardNotAvailableException(e)) { - attemptCount.incrementAndGet(); - indexBwcInvalidation(userToken, listener, attemptCount, expirationEpochMilli); - } else { - listener.onFailure(e); - } - }), client::index)); + bulkRequestBuilder.add(indexRequest); + } + bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + final BulkRequest bulkRequest = bulkRequestBuilder.request(); + securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), + () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, + ActionListener.wrap(bulkResponse -> { + List retryTokenIds = new ArrayList<>(); + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + if (bulkItemResponse.isFailed()) { + Throwable cause = bulkItemResponse.getFailure().getCause(); + logger.error(cause.getMessage()); + traceLog("(bwc) invalidate tokens", cause); + if (isShardNotAvailableException(cause)) { + retryTokenIds.add(getTokenIdFromInvalidatedTokenDocumentId(bulkItemResponse.getFailure().getId())); + } else if ((cause instanceof VersionConflictEngineException) == false){ + // We don't handle VersionConflictEngineException, the ticket has been invalidated + listener.onFailure(bulkItemResponse.getFailure().getCause()); + } + } + } + if (retryTokenIds.isEmpty() == false) { + attemptCount.incrementAndGet(); + indexBwcInvalidation(retryTokenIds, listener, attemptCount, expirationEpochMilli, previousResult); + } + indexInvalidation(tokenIds, listener, attemptCount, "access_token", previousResult); + }, e -> { + Throwable cause = ExceptionsHelper.unwrapCause(e); + traceLog("(bwc) invalidate tokens", cause); + if (isShardNotAvailableException(cause)) { + attemptCount.incrementAndGet(); + indexBwcInvalidation(tokenIds, listener, attemptCount, expirationEpochMilli, previousResult); + } else { + listener.onFailure(e); + } + }), + client::bulk)); } } /** - * Performs the actual invalidation of a token + * Performs the actual invalidation of a collection of tokens * - * @param tokenDocId the id of the token doc to invalidate + * @param tokenIds the tokens to invalidate * @param listener the listener to notify upon completion * @param attemptCount the number of attempts to invalidate that have already been tried - * @param srcPrefix the prefix to use when constructing the doc to update - * @param documentVersion the expected version of the document we will update + * @param srcPrefix the prefix to use when constructing the doc to update, either refresh_token or access_token depending on + * what type of tokens should be invalidated + * @param previousResult if this not the initial attempt for invalidation, it contains the result of invalidating + * tokens up to the point of the retry. This result is added to the result of the current attempt */ - private void indexInvalidation(String tokenDocId, Version version, ActionListener listener, AtomicInteger attemptCount, - String srcPrefix, long documentVersion) { - if (attemptCount.get() > MAX_RETRY_ATTEMPTS) { - logger.warn("Failed to invalidate token [{}] after [{}] attempts", tokenDocId, attemptCount.get()); - listener.onFailure(invalidGrantException("failed to invalidate token")); + private void indexInvalidation(Collection tokenIds, ActionListener listener, + AtomicInteger attemptCount, String srcPrefix, @Nullable TokensInvalidationResult previousResult) { + if (tokenIds.isEmpty()) { + logger.warn("No [{}] tokens provided for invalidation", srcPrefix); + listener.onFailure(invalidGrantException("No tokens provided for invalidation")); + } else if (attemptCount.get() > MAX_RETRY_ATTEMPTS) { + logger.warn("Failed to invalidate [{}] tokens after [{}] attempts", tokenIds.size(), + attemptCount.get()); + listener.onFailure(invalidGrantException("failed to invalidate tokens")); } else { - UpdateRequest request = client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId) + BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + for (String tokenId : tokenIds) { + UpdateRequest request = client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(tokenId)) .setDoc(srcPrefix, Collections.singletonMap("invalidated", true)) - .setVersion(documentVersion) - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + .setFetchSource(srcPrefix, null) .request(); - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", tokenDocId, ex)), - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, - ActionListener.wrap(updateResponse -> { - logger.debug("Invalidated [{}] for doc [{}]", srcPrefix, tokenDocId); - if (updateResponse.getGetResult() != null - && updateResponse.getGetResult().sourceAsMap().containsKey(srcPrefix) - && ((Map) updateResponse.getGetResult().sourceAsMap().get(srcPrefix)) - .containsKey("invalidated")) { - final boolean prevInvalidated = (boolean) - ((Map) updateResponse.getGetResult().sourceAsMap().get(srcPrefix)) - .get("invalidated"); - listener.onResponse(prevInvalidated == false); - } else { - listener.onResponse(true); + bulkRequestBuilder.add(request); + } + bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), + () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequestBuilder.request(), + ActionListener.wrap(bulkResponse -> { + ArrayList retryTokenDocIds = new ArrayList<>(); + ArrayList failedRequestResponses = new ArrayList<>(); + ArrayList previouslyInvalidated = new ArrayList<>(); + ArrayList invalidated = new ArrayList<>(); + if (null != previousResult) { + failedRequestResponses.addAll((previousResult.getErrors())); + previouslyInvalidated.addAll(previousResult.getPreviouslyInvalidatedTokens()); + invalidated.addAll(previousResult.getInvalidatedTokens()); } - }, e -> { - Throwable cause = ExceptionsHelper.unwrapCause(e); - traceLog("invalidate token", tokenDocId, cause); - if (cause instanceof DocumentMissingException) { - if (version.onOrAfter(Version.V_6_2_0)) { - // the document should always be there! - listener.onFailure(e); + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + if (bulkItemResponse.isFailed()) { + Throwable cause = bulkItemResponse.getFailure().getCause(); + final String failedTokenDocId = getTokenIdFromDocumentId(bulkItemResponse.getFailure().getId()); + if (isShardNotAvailableException(cause)) { + retryTokenDocIds.add(failedTokenDocId); + } + else { + traceLog("invalidate access token", failedTokenDocId, cause); + failedRequestResponses.add(new ElasticsearchException("Error invalidating " + srcPrefix + ": ", cause)); + } } else { - listener.onResponse(false); + UpdateResponse updateResponse = bulkItemResponse.getResponse(); + if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) { + logger.debug("Invalidated [{}] for doc [{}]", srcPrefix, updateResponse.getGetResult().getId()); + invalidated.add(updateResponse.getGetResult().getId()); + } else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) { + previouslyInvalidated.add(updateResponse.getGetResult().getId()); + } } - } else if (cause instanceof VersionConflictEngineException - || isShardNotAvailableException(cause)) { + } + if (retryTokenDocIds.isEmpty() == false) { + TokensInvalidationResult incompleteResult = new TokensInvalidationResult(invalidated, previouslyInvalidated, + failedRequestResponses, attemptCount.get()); attemptCount.incrementAndGet(); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(), - ActionListener.wrap(getResult -> { - if (getResult.isExists()) { - Map source = getResult.getSource(); - Map accessTokenSource = (Map) source.get("access_token"); - Consumer onFailure = ex -> listener.onFailure(traceLog("get token", tokenDocId, ex)); - if (accessTokenSource == null) { - onFailure.accept(new IllegalArgumentException( - "token document is missing access_token field")); - } else { - Boolean invalidated = (Boolean) accessTokenSource.get("invalidated"); - if (invalidated == null) { - onFailure.accept(new IllegalStateException( - "token document missing invalidated value")); - } else if (invalidated) { - logger.trace("Token [{}] is already invalidated", tokenDocId); - listener.onResponse(false); - } else { - indexInvalidation(tokenDocId, version, listener, attemptCount, srcPrefix, - getResult.getVersion()); - } - } - } else if (version.onOrAfter(Version.V_6_2_0)) { - logger.warn("could not find token document [{}] but there should " + - "be one as token has version [{}]", tokenDocId, version); - listener.onFailure(invalidGrantException("could not invalidate the token")); - } else { - listener.onResponse(false); - } - }, - e1 -> { - traceLog("get token", tokenDocId, e1); - if (isShardNotAvailableException(e1)) { - // don't increment count; call again - indexInvalidation(tokenDocId, version, listener, attemptCount, srcPrefix, - documentVersion); - } else { - listener.onFailure(e1); - } - }), client::get); + indexInvalidation(retryTokenDocIds, listener, attemptCount, srcPrefix, incompleteResult); + } + TokensInvalidationResult result = new TokensInvalidationResult(invalidated, previouslyInvalidated, + failedRequestResponses, attemptCount.get()); + listener.onResponse(result); + }, e -> { + Throwable cause = ExceptionsHelper.unwrapCause(e); + traceLog("invalidate tokens", cause); + if (isShardNotAvailableException(cause)) { + attemptCount.incrementAndGet(); + indexInvalidation(tokenIds, listener, attemptCount, srcPrefix, previousResult); } else { listener.onFailure(e); } - }), client::update)); + }), client::bulk)); } } @@ -676,12 +761,12 @@ private void indexInvalidation(String tokenDocId, Version version, ActionListene public void refreshToken(String refreshToken, ActionListener> listener) { ensureEnabled(); findTokenFromRefreshToken(refreshToken, - ActionListener.wrap(tuple -> { - final Authentication userAuth = Authentication.readFromContext(client.threadPool().getThreadContext()); - final String tokenDocId = tuple.v1().getHits().getHits()[0].getId(); - innerRefresh(tokenDocId, userAuth, listener, tuple.v2()); - }, listener::onFailure), - new AtomicInteger(0)); + ActionListener.wrap(tuple -> { + final Authentication userAuth = Authentication.readFromContext(client.threadPool().getThreadContext()); + final String tokenDocId = tuple.v1().getHits().getHits()[0].getId(); + innerRefresh(tokenDocId, userAuth, listener, tuple.v2()); + }, listener::onFailure), + new AtomicInteger(0)); } private void findTokenFromRefreshToken(String refreshToken, ActionListener> listener, @@ -691,11 +776,11 @@ private void findTokenFromRefreshToken(String refreshToken, ActionListener checkClient(Map } /** - * Find all stored refresh and access tokens that have not been invalidated or expired, and were issued against + * Find stored refresh and access tokens that have not been invalidated or expired, and were issued against * the specified realm. + * + * @param realmName The name of the realm for which to get the tokens + * @param listener The listener to notify upon completion + * @param filter an optional Predicate to test the source of the found documents against */ - public void findActiveTokensForRealm(String realmName, ActionListener>> listener) { + public void findActiveTokensForRealm(String realmName, ActionListener>> listener, + @Nullable Predicate> filter) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); if (Strings.isNullOrEmpty(realmName)) { listener.onFailure(new IllegalArgumentException("Realm name is required")); @@ -883,7 +972,10 @@ public void findActiveTokensForRealm(String realmName, ActionListener ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, filter))); + } + } + + /** + * Find stored refresh and access tokens that have not been invalidated or expired, and were issued for + * the specified user. + * + * @param username The user for which to get the tokens + * @param listener The listener to notify upon completion + */ + public void findActiveTokensForUser(String username, ActionListener>> listener) { + ensureEnabled(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + if (Strings.isNullOrEmpty(username)) { + listener.onFailure(new IllegalArgumentException("username is required")); + } else if (frozenSecurityIndex.indexExists() == false) { + listener.onResponse(Collections.emptyList()); + } else if (frozenSecurityIndex.isAvailable() == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else { + final Instant now = clock.instant(); + final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("doc_type", "token")) + .filter(QueryBuilders.boolQuery() + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("access_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("access_token.user_token.expiration_time").gte(now.toEpochMilli())) + ) + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("refresh_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("creation_time").gte(now.toEpochMilli() - TimeValue.timeValueHours(24).millis())) + ) + ); + + final SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) + .setQuery(boolQuery) + .setVersion(false) + .setSize(1000) + .setFetchSource(true) + .request(); securityIndex.checkIndexVersionThenExecute(listener::onFailure, - () -> ScrollHelper.fetchAllByEntity(client, request, listener, this::parseHit)); + () -> ScrollHelper.fetchAllByEntity(client, request, listener, + (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); } } - private Tuple parseHit(SearchHit hit) { + private static Predicate> isOfUser(String username) { + return source -> { + String auth = (String) source.get("authentication"); + Integer version = (Integer) source.get("version"); + Version authVersion = Version.fromId(version); + try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(auth))) { + in.setVersion(authVersion); + Authentication authentication = new Authentication(in); + return authentication.getUser().principal().equals(username); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + + private Tuple filterAndParseHit(SearchHit hit, @Nullable Predicate> filter) { final Map source = hit.getSourceAsMap(); if (source == null) { throw new IllegalStateException("token document did not have source but source should have been fetched"); } - try { - return parseTokensFromDocument(source); + return parseTokensFromDocument(source, filter); } catch (IOException e) { throw invalidGrantException("cannot read token from document"); } } /** - * @return A {@link Tuple} of access-token and refresh-token-id + * + * Parses a token document into a Tuple of a {@link UserToken} and a String representing the corresponding refresh_token + * + * @param source The token document source as retrieved + * @param filter an optional Predicate to test the source of the UserToken against + * @return A {@link Tuple} of access-token and refresh-token-id or null if a Predicate is defined and the userToken source doesn't + * satisfy it */ - private Tuple parseTokensFromDocument(Map source) throws IOException { - final String refreshToken = (String) ((Map) source.get("refresh_token")).get("token"); + private Tuple parseTokensFromDocument(Map source, @Nullable Predicate> filter) + throws IOException { + final String refreshToken = (String) ((Map) source.get("refresh_token")).get("token"); final Map userTokenSource = (Map) - ((Map) source.get("access_token")).get("user_token"); + ((Map) source.get("access_token")).get("user_token"); + if (null != filter && filter.test(userTokenSource) == false) { + return null; + } final String id = (String) userTokenSource.get("id"); final Integer version = (Integer) userTokenSource.get("version"); final String authString = (String) userTokenSource.get("authentication"); @@ -951,6 +1112,23 @@ private static String getTokenDocumentId(String id) { return "token_" + id; } + private static String getTokenIdFromDocumentId(String docId) { + if (docId.startsWith("token_") == false) { + throw new IllegalStateException("TokenDocument ID [" + docId + "] has unexpected value"); + } else { + return docId.substring("token_".length()); + } + } + + private static String getTokenIdFromInvalidatedTokenDocumentId(String docId) { + final String invalidatedTokenDocPrefix = INVALIDATED_TOKEN_DOC_TYPE + "_"; + if (docId.startsWith(invalidatedTokenDocPrefix) == false) { + throw new IllegalStateException("InvalidatedTokenDocument ID [" + docId + "] has unexpected value"); + } else { + return docId.substring(invalidatedTokenDocPrefix.length()); + } + } + private void ensureEnabled() { if (enabled == false) { throw new IllegalStateException("tokens are not enabled"); @@ -1149,7 +1327,7 @@ private static ElasticsearchSecurityException expiredTokenException() { } /** - * Creates an {@link ElasticsearchSecurityException} that indicates the token was expired. It + * Creates an {@link ElasticsearchSecurityException} that indicates the token was malformed. It * is up to the client to re-authenticate and obtain a new token. The format for this response * is defined in */ @@ -1171,7 +1349,7 @@ private static ElasticsearchSecurityException invalidGrantException(String detai } /** - * Logs an exception at TRACE level (if enabled) + * Logs an exception concerning a specific Token at TRACE level (if enabled) */ private E traceLog(String action, String identifier, E exception) { if (logger.isTraceEnabled()) { @@ -1179,12 +1357,34 @@ private E traceLog(String action, String identifier, E exc final ElasticsearchException esEx = (ElasticsearchException) exception; final Object detail = esEx.getHeader("error_description"); if (detail != null) { - logger.trace("Failure in [{}] for id [{}] - [{}] [{}]", action, identifier, detail, esEx.getDetailedMessage()); + logger.trace(() -> new ParameterizedMessage("Failure in [{}] for id [{}] - [{}]", action, identifier, detail), + esEx); + } else { + logger.trace(() -> new ParameterizedMessage("Failure in [{}] for id [{}]", action, identifier), + esEx); + } + } else { + logger.trace(() -> new ParameterizedMessage("Failure in [{}] for id [{}]", action, identifier), exception); + } + } + return exception; + } + + /** + * Logs an exception at TRACE level (if enabled) + */ + private E traceLog(String action, E exception) { + if (logger.isTraceEnabled()) { + if (exception instanceof ElasticsearchException) { + final ElasticsearchException esEx = (ElasticsearchException) exception; + final Object detail = esEx.getHeader("error_description"); + if (detail != null) { + logger.trace(() -> new ParameterizedMessage("Failure in [{}] - [{}]", action, detail), esEx); } else { - logger.trace("Failure in [{}] for id [{}] - [{}]", action, identifier, esEx.getDetailedMessage()); + logger.trace(() -> new ParameterizedMessage("Failure in [{}]", action), esEx); } } else { - logger.trace("Failure in [{}] for id [{}] - [{}]", action, identifier, exception.toString()); + logger.trace(() -> new ParameterizedMessage("Failure in [{}]", action), exception); } } return exception; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java index 52228d2823a68..9801f3c93c839 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -37,11 +36,32 @@ public final class RestInvalidateTokenAction extends SecurityBaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestInvalidateTokenAction.class)); - static final ConstructingObjectParser, Void> PARSER = - new ConstructingObjectParser<>("invalidate_token", a -> new Tuple<>((String) a[0], (String) a[1])); + static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("invalidate_token", a -> { + final String token = (String) a[0]; + final String refreshToken = (String) a[1]; + final String tokenString; + final String tokenType; + if (Strings.hasLength(token) && Strings.hasLength(refreshToken)) { + throw new IllegalArgumentException("only one of [token, refresh_token] may be sent per request"); + } else if (Strings.hasLength(token)) { + tokenString = token; + tokenType = InvalidateTokenRequest.Type.ACCESS_TOKEN.getValue(); + } else if (Strings.hasLength(refreshToken)) { + tokenString = refreshToken; + tokenType = InvalidateTokenRequest.Type.REFRESH_TOKEN.getValue(); + } else { + tokenString = null; + tokenType = null; + } + return new InvalidateTokenRequest(tokenString, tokenType, (String) a[2], (String) a[3]); + }); + static { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("token")); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("refresh_token")); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("realm_name")); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("username")); } public RestInvalidateTokenAction(Settings settings, RestController controller, XPackLicenseState xPackLicenseState) { @@ -60,36 +80,16 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { try (XContentParser parser = request.contentParser()) { - final Tuple tuple = PARSER.parse(parser, null); - final String token = tuple.v1(); - final String refreshToken = tuple.v2(); - - final String tokenString; - final InvalidateTokenRequest.Type type; - if (Strings.hasLength(token) && Strings.hasLength(refreshToken)) { - throw new IllegalArgumentException("only one of [token, refresh_token] may be sent per request"); - } else if (Strings.hasLength(token)) { - tokenString = token; - type = InvalidateTokenRequest.Type.ACCESS_TOKEN; - } else if (Strings.hasLength(refreshToken)) { - tokenString = refreshToken; - type = InvalidateTokenRequest.Type.REFRESH_TOKEN; - } else { - tokenString = null; - type = null; - } - - final InvalidateTokenRequest tokenRequest = new InvalidateTokenRequest(tokenString, type); - return channel -> client.execute(InvalidateTokenAction.INSTANCE, tokenRequest, - new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(InvalidateTokenResponse invalidateResp, - XContentBuilder builder) throws Exception { - return new BytesRestResponse(RestStatus.OK, builder.startObject() - .field("created", invalidateResp.isCreated()) - .endObject()); - } - }); + final InvalidateTokenRequest invalidateTokenRequest = PARSER.parse(parser, null); + return channel -> client.execute(InvalidateTokenAction.INSTANCE, invalidateTokenRequest, + new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(InvalidateTokenResponse invalidateResp, + XContentBuilder builder) throws Exception { + invalidateResp.toXContent(builder, channel.request()); + return new BytesRestResponse(RestStatus.OK, builder); + } + }); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index ba1d1762f0654..5a4c8f3bde824 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -11,6 +11,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -21,11 +25,11 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; @@ -106,11 +110,12 @@ public class TransportSamlInvalidateSessionActionTests extends SamlTestCase { private SamlRealm samlRealm; private TokenService tokenService; private List indexRequests; - private List updateRequests; + private List bulkRequests; private List searchRequests; private TransportSamlInvalidateSessionAction action; private SamlLogoutRequestHandler.Result logoutRequest; private Function searchFunction = ignore -> new SearchHit[0]; + private Function searchScrollFunction = ignore -> new SearchHit[0]; @Before public void setup() throws Exception { @@ -132,8 +137,8 @@ public void setup() throws Exception { new Authentication(new User("kibana"), new RealmRef("realm", "type", "node"), null).writeToContext(threadContext); indexRequests = new ArrayList<>(); - updateRequests = new ArrayList<>(); searchRequests = new ArrayList<>(); + bulkRequests = new ArrayList<>(); final Client client = new NoOpClient(threadPool) { @Override protected @@ -143,20 +148,29 @@ void doExecute(Action action, Request request, ActionListener findTokenByRefreshToken(SearchHit[] searchHits) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java index 66d3233b07a95..7dec105e1ee80 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java @@ -6,7 +6,11 @@ package org.elasticsearch.xpack.security.action.saml; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; @@ -24,7 +28,6 @@ import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.MapBuilder; @@ -72,6 +75,9 @@ import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey; import static org.elasticsearch.xpack.security.authc.TokenServiceTests.mockGetTokenFromId; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; @@ -89,7 +95,7 @@ public class TransportSamlLogoutActionTests extends SamlTestCase { private SamlRealm samlRealm; private TokenService tokenService; private List indexRequests; - private List updateRequests; + private List bulkRequests; private TransportSamlLogoutAction action; private Client client; @@ -112,7 +118,7 @@ public void setup() throws Exception { new Authentication(new User("kibana"), new Authentication.RealmRef("realm", "type", "node"), null).writeToContext(threadContext); indexRequests = new ArrayList<>(); - updateRequests = new ArrayList<>(); + bulkRequests = new ArrayList<>(); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); when(client.settings()).thenReturn(settings); @@ -137,6 +143,10 @@ public void setup() throws Exception { .setId((String) invocationOnMock.getArguments()[2]); return builder; }).when(client).prepareUpdate(anyString(), anyString(), anyString()); + doAnswer(invocationOnMock -> { + BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); + return builder; + }).when(client).prepareBulk(); when(client.prepareMultiGet()).thenReturn(new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE)); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; @@ -154,15 +164,6 @@ public void setup() throws Exception { listener.onResponse(response); return Void.TYPE; }).when(client).multiGet(any(MultiGetRequest.class), any(ActionListener.class)); - doAnswer(invocationOnMock -> { - UpdateRequest updateRequest = (UpdateRequest) invocationOnMock.getArguments()[0]; - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - updateRequests.add(updateRequest); - final UpdateResponse response = new UpdateResponse( - updateRequest.getShardId(), updateRequest.type(), updateRequest.id(), 1, DocWriteResponse.Result.UPDATED); - listener.onResponse(response); - return Void.TYPE; - }).when(client).update(any(UpdateRequest.class), any(ActionListener.class)); doAnswer(invocationOnMock -> { IndexRequest indexRequest = (IndexRequest) invocationOnMock.getArguments()[0]; ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; @@ -181,6 +182,14 @@ public void setup() throws Exception { listener.onResponse(response); return Void.TYPE; }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class)); + doAnswer(invocationOnMock -> { + BulkRequest bulkRequest = (BulkRequest) invocationOnMock.getArguments()[0]; + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + bulkRequests.add(bulkRequest); + final BulkResponse response = new BulkResponse(new BulkItemResponse[0], 1); + listener.onResponse(response); + return Void.TYPE; + }).when(client).bulk(any(BulkRequest.class), any(ActionListener.class)); final SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); doAnswer(inv -> { @@ -247,9 +256,17 @@ public void testLogoutInvalidatesToken() throws Exception { assertThat(indexRequest1, notNullValue()); assertThat(indexRequest1.id(), startsWith("token")); - final IndexRequest indexRequest2 = indexRequests.get(1); - assertThat(indexRequest2, notNullValue()); - assertThat(indexRequest2.id(), startsWith("invalidated-token")); + assertThat(bulkRequests.size(), equalTo(2)); + final BulkRequest bulkRequest1 = bulkRequests.get(0); + assertThat(bulkRequest1.requests().size(), equalTo(1)); + assertThat(bulkRequest1.requests().get(0), instanceOf(IndexRequest.class)); + assertThat(bulkRequest1.requests().get(0).id(), startsWith("invalidated-token_")); + + final BulkRequest bulkRequest2 = bulkRequests.get(1); + assertThat(bulkRequest2.requests().size(), equalTo(1)); + assertThat(bulkRequest2.requests().get(0), instanceOf(UpdateRequest.class)); + assertThat(bulkRequest2.requests().get(0).id(), startsWith("token_")); + assertThat(bulkRequest2.requests().get(0).toString(), containsString("\"access_token\":{\"invalidated\":true")); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index c4efdc16e10e6..968c17f556b9b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -144,7 +144,9 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { .prepareInvalidateToken(response.getTokenString()) .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN) .get(); - assertTrue(invalidateResponse.isCreated()); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1)); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) @@ -189,6 +191,72 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { }, 30, TimeUnit.SECONDS); } + public void testInvalidateAllTokensForUser() throws Exception{ + final int numOfRequests = randomIntBetween(5, 10); + for (int i = 0; i < numOfRequests; i++) { + securityClient().prepareCreateToken() + .setGrantType("password") + .setUsername(SecuritySettingsSource.TEST_USER_NAME) + .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())) + .get(); + } + Client client = client().filterWithHeader(Collections.singletonMap("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); + SecurityClient securityClientSuperuser = new SecurityClient(client); + InvalidateTokenResponse invalidateResponse = securityClientSuperuser + .prepareInvalidateToken() + .setUserName(SecuritySettingsSource.TEST_USER_NAME) + .get(); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(2 * (numOfRequests))); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); + } + + public void testInvalidateAllTokensForRealm() throws Exception{ + final int numOfRequests = randomIntBetween(5, 10); + for (int i = 0; i < numOfRequests; i++) { + securityClient().prepareCreateToken() + .setGrantType("password") + .setUsername(SecuritySettingsSource.TEST_USER_NAME) + .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())) + .get(); + } + Client client = client().filterWithHeader(Collections.singletonMap("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); + SecurityClient securityClientSuperuser = new SecurityClient(client); + InvalidateTokenResponse invalidateResponse = securityClientSuperuser + .prepareInvalidateToken() + .setRealmName("file") + .get(); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(2 * (numOfRequests))); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); + } + + public void testInvalidateAllTokensForRealmThatHasNone() { + final int numOfRequests = randomIntBetween(2, 4); + for (int i = 0; i < numOfRequests; i++) { + securityClient().prepareCreateToken() + .setGrantType("password") + .setUsername(SecuritySettingsSource.TEST_USER_NAME) + .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())) + .get(); + } + Client client = client().filterWithHeader(Collections.singletonMap("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); + SecurityClient securityClientSuperuser = new SecurityClient(client); + InvalidateTokenResponse invalidateResponse = securityClientSuperuser + .prepareInvalidateToken() + .setRealmName("saml") + .get(); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); + } + public void testExpireMultipleTimes() { CreateTokenResponse response = securityClient().prepareCreateToken() .setGrantType("password") @@ -200,12 +268,16 @@ public void testExpireMultipleTimes() { .prepareInvalidateToken(response.getTokenString()) .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN) .get(); - assertTrue(invalidateResponse.isCreated()); - assertFalse(securityClient() - .prepareInvalidateToken(response.getTokenString()) - .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN) - .get() - .isCreated()); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1)); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); + InvalidateTokenResponse invalidateAgainResponse = securityClient() + .prepareInvalidateToken(response.getTokenString()) + .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN) + .get(); + assertThat(invalidateAgainResponse.getResult().getInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateAgainResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(1)); + assertThat(invalidateAgainResponse.getResult().getErrors().size(), equalTo(0)); } public void testRefreshingToken() { @@ -248,7 +320,9 @@ public void testRefreshingInvalidatedToken() { .prepareInvalidateToken(createTokenResponse.getRefreshToken()) .setType(InvalidateTokenRequest.Type.REFRESH_TOKEN) .get(); - assertTrue(invalidateResponse.isCreated()); + assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1)); + assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get()); @@ -362,9 +436,11 @@ public void testClientCredentialsGrant() throws Exception { // invalidate PlainActionFuture invalidateResponseFuture = new PlainActionFuture<>(); InvalidateTokenRequest invalidateTokenRequest = - new InvalidateTokenRequest(createTokenResponse.getTokenString(), InvalidateTokenRequest.Type.ACCESS_TOKEN); + new InvalidateTokenRequest(createTokenResponse.getTokenString(), InvalidateTokenRequest.Type.ACCESS_TOKEN.getValue()); securityClient.invalidateToken(invalidateTokenRequest, invalidateResponseFuture); - assertTrue(invalidateResponseFuture.get().isCreated()); + assertThat(invalidateResponseFuture.get().getResult().getInvalidatedTokens().size(), equalTo(1)); + assertThat(invalidateResponseFuture.get().getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0)); + assertThat(invalidateResponseFuture.get().getResult().getErrors().size(), equalTo(0)); ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> { PlainActionFuture responseFuture = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 7926b44a38cb8..286f07667eca6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -523,7 +524,7 @@ public void testTokenServiceDisabled() throws Exception { assertNull(future.get()); e = expectThrows(IllegalStateException.class, () -> { - PlainActionFuture invalidateFuture = new PlainActionFuture<>(); + PlainActionFuture invalidateFuture = new PlainActionFuture<>(); tokenService.invalidateAccessToken((String) null, invalidateFuture); invalidateFuture.actionGet(); }); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/TokensInvalidationResultTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/TokensInvalidationResultTests.java new file mode 100644 index 0000000000000..06c9411d0bc1f --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/TokensInvalidationResultTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.security.authc.support; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; + +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class TokensInvalidationResultTests extends ESTestCase { + + public void testToXcontent() throws Exception{ + TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList("token1", "token2"), + Arrays.asList("token3", "token4"), + Arrays.asList(new ElasticsearchException("foo", new IllegalStateException("bar")), + new ElasticsearchException("boo", new IllegalStateException("far"))), + randomIntBetween(0, 5)); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), + equalTo( + "{\"created\":false," + + "\"invalidated_tokens\":2," + + "\"previously_invalidated_tokens\":2," + + "\"error_count\":2," + + "\"error_details\":[" + + "{\"type\":\"exception\"," + + "\"reason\":\"foo\"," + + "\"caused_by\":{" + + "\"type\":\"illegal_state_exception\"," + + "\"reason\":\"bar\"" + + "}" + + "}," + + "{\"type\":\"exception\"," + + "\"reason\":\"boo\"," + + "\"caused_by\":{" + + "\"type\":\"illegal_state_exception\"," + + "\"reason\":\"far\"" + + "}" + + "}" + + "]" + + "}")); + } + } + + public void testToXcontentWithNoErrors() throws Exception{ + TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList("token1", "token2"), + Collections.emptyList(), + Collections.emptyList(), randomIntBetween(0, 5)); + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), + equalTo( + "{\"created\":true," + + "\"invalidated_tokens\":2," + + "\"previously_invalidated_tokens\":0," + + "\"error_count\":0" + + "}")); + } + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenActionTests.java new file mode 100644 index 0000000000000..00850ba6e5ad6 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenActionTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.rest.action.oauth2; + +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequest; + +import static org.hamcrest.Matchers.containsString; + +public class RestInvalidateTokenActionTests extends ESTestCase { + + public void testParserForUserAndRealm() throws Exception { + final String request = "{" + + "\"username\": \"user1\"," + + "\"realm_name\": \"realm1\"" + + "}"; + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, request)) { + InvalidateTokenRequest invalidateTokenRequest = RestInvalidateTokenAction.PARSER.parse(parser, null); + assertEquals("user1", invalidateTokenRequest.getUserName()); + assertEquals("realm1", invalidateTokenRequest.getRealmName()); + assertNull(invalidateTokenRequest.getTokenString()); + assertNull(invalidateTokenRequest.getTokenType()); + } + } + + public void testParserForToken() throws Exception { + final String request = "{" + + "\"refresh_token\": \"refresh_token_string\"" + + "}"; + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, request)) { + InvalidateTokenRequest invalidateTokenRequest = RestInvalidateTokenAction.PARSER.parse(parser, null); + assertEquals("refresh_token_string", invalidateTokenRequest.getTokenString()); + assertEquals("refresh_token", invalidateTokenRequest.getTokenType().getValue()); + assertNull(invalidateTokenRequest.getRealmName()); + assertNull(invalidateTokenRequest.getUserName()); + } + } + + public void testParserForIncorrectInput() throws Exception { + final String request = "{" + + "\"refresh_token\": \"refresh_token_string\"," + + "\"token\": \"access_token_string\"" + + "}"; + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, request)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestInvalidateTokenAction.PARSER.parse(parser, + null)); + assertThat(e.getCause().getMessage(), containsString("only one of [token, refresh_token] may be sent per request")); + + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/token/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/token/10_basic.yml index 43f25a11db07a..81389ac8524f4 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/token/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/token/10_basic.yml @@ -5,7 +5,7 @@ setup: - do: cluster.health: - wait_for_status: yellow + wait_for_status: yellow - do: security.put_user: @@ -79,7 +79,93 @@ teardown: body: token: $token - - match: { created: true } + - match: { created: true} + - match: { invalidated_tokens: 1 } + - match: { previously_invalidated_tokens: 0 } + - match: { error_count: 0 } + + - do: + catch: unauthorized + headers: + Authorization: Bearer ${token} + security.authenticate: {} + +--- +"Test invalidate user's tokens": + + - do: + security.get_token: + body: + grant_type: "password" + username: "token_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + + - do: + security.invalidate_token: + body: + username: "token_user" + + - match: { created: true} + - match: { invalidated_tokens: 2 } + - match: { previously_invalidated_tokens: 0 } + - match: { error_count: 0 } + + - do: + catch: unauthorized + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + +--- +"Test invalidate realm's tokens": + + - do: + security.get_token: + body: + grant_type: "password" + username: "token_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + + - do: + security.invalidate_token: + body: + realm_name: "default_native" + + - match: { created: true} + - match: { invalidated_tokens: 2 } + - match: { previously_invalidated_tokens: 0 } + - match: { error_count: 0 } - do: catch: unauthorized From fe0c22a4c961a4bda535d94276d4783781ac2d9d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 18 Dec 2018 09:20:51 +0100 Subject: [PATCH 52/57] Add raw sort values to SearchSortValues transport serialization (#36617) In order for CCS alternate execution mode (see #32125) to be able to do the final reduction step on the CCS coordinating node, we need to serialize additional info in the transport layer as part of each `SearchHit`. Sort values are already present but they are formatted according to the provided `DocValueFormat` provided. The CCS node needs to be able to reconstruct the lucene `FieldDoc` to include in the `TopFieldDocs` and `CollapseTopFieldDocs` which will feed the `mergeTopDocs` method used to reduce multiple search responses (one per cluster) into one. This commit adds such information to the `SearchSortValues` and exposes it through a new getter method added to `SearchHit` for retrieval. This info is only serialized at transport and never printed out at REST. --- .../org/elasticsearch/search/SearchHit.java | 31 ++-- .../search/SearchSortValues.java | 134 +++++++----------- .../common/lucene/LuceneTests.java | 20 +-- .../search/SearchSortValuesTests.java | 58 +++++--- .../test/AbstractWireSerializingTestCase.java | 2 +- 5 files changed, 125 insertions(+), 120 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 3d8ea3845464f..7fd68852ce284 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -19,16 +19,6 @@ package org.elasticsearch.search; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - import org.apache.lucene.search.Explanation; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.OriginalIndices; @@ -61,6 +51,16 @@ import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.transport.RemoteClusterAware; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static java.util.Collections.unmodifiableMap; @@ -311,10 +311,17 @@ public void sortValues(SearchSortValues sortValues) { } /** - * An array of the sort values used. + * An array of the (formatted) sort values used. */ public Object[] getSortValues() { - return sortValues.sortValues(); + return sortValues.getFormattedSortValues(); + } + + /** + * An array of the (raw) sort values used. + */ + public Object[] getRawSortValues() { + return sortValues.getRawSortValues(); } /** diff --git a/server/src/main/java/org/elasticsearch/search/SearchSortValues.java b/server/src/main/java/org/elasticsearch/search/SearchSortValues.java index c79b5ad74d785..1382966544430 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchSortValues.java +++ b/server/src/main/java/org/elasticsearch/search/SearchSortValues.java @@ -20,9 +20,11 @@ package org.elasticsearch.search; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,101 +37,56 @@ public class SearchSortValues implements ToXContentFragment, Writeable { - static final SearchSortValues EMPTY = new SearchSortValues(new Object[0]); - private final Object[] sortValues; + private static final Object[] EMPTY_ARRAY = new Object[0]; + static final SearchSortValues EMPTY = new SearchSortValues(EMPTY_ARRAY); + + private final Object[] formattedSortValues; + private final Object[] rawSortValues; SearchSortValues(Object[] sortValues) { - this.sortValues = Objects.requireNonNull(sortValues, "sort values must not be empty"); + this.formattedSortValues = Objects.requireNonNull(sortValues, "sort values must not be empty"); + this.rawSortValues = EMPTY_ARRAY; } - public SearchSortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) { - Objects.requireNonNull(sortValues); + public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormats) { + Objects.requireNonNull(rawSortValues); Objects.requireNonNull(sortValueFormats); - this.sortValues = Arrays.copyOf(sortValues, sortValues.length); - for (int i = 0; i < sortValues.length; ++i) { - if (this.sortValues[i] instanceof BytesRef) { - this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]); + if (rawSortValues.length != sortValueFormats.length) { + throw new IllegalArgumentException("formattedSortValues and sortValueFormats must hold the same number of items"); + } + this.rawSortValues = rawSortValues; + this.formattedSortValues = Arrays.copyOf(rawSortValues, rawSortValues.length); + for (int i = 0; i < rawSortValues.length; ++i) { + //we currently format only BytesRef but we may want to change that in the future + Object sortValue = rawSortValues[i]; + if (sortValue instanceof BytesRef) { + this.formattedSortValues[i] = sortValueFormats[i].format((BytesRef) sortValue); } } } - public SearchSortValues(StreamInput in) throws IOException { - int size = in.readVInt(); - if (size > 0) { - sortValues = new Object[size]; - for (int i = 0; i < sortValues.length; i++) { - byte type = in.readByte(); - if (type == 0) { - sortValues[i] = null; - } else if (type == 1) { - sortValues[i] = in.readString(); - } else if (type == 2) { - sortValues[i] = in.readInt(); - } else if (type == 3) { - sortValues[i] = in.readLong(); - } else if (type == 4) { - sortValues[i] = in.readFloat(); - } else if (type == 5) { - sortValues[i] = in.readDouble(); - } else if (type == 6) { - sortValues[i] = in.readByte(); - } else if (type == 7) { - sortValues[i] = in.readShort(); - } else if (type == 8) { - sortValues[i] = in.readBoolean(); - } else { - throw new IOException("Can't match type [" + type + "]"); - } - } + SearchSortValues(StreamInput in) throws IOException { + this.formattedSortValues = in.readArray(Lucene::readSortValue, Object[]::new); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + this.rawSortValues = in.readArray(Lucene::readSortValue, Object[]::new); } else { - sortValues = new Object[0]; + this.rawSortValues = EMPTY_ARRAY; } } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(sortValues.length); - for (Object sortValue : sortValues) { - if (sortValue == null) { - out.writeByte((byte) 0); - } else { - Class type = sortValue.getClass(); - if (type == String.class) { - out.writeByte((byte) 1); - out.writeString((String) sortValue); - } else if (type == Integer.class) { - out.writeByte((byte) 2); - out.writeInt((Integer) sortValue); - } else if (type == Long.class) { - out.writeByte((byte) 3); - out.writeLong((Long) sortValue); - } else if (type == Float.class) { - out.writeByte((byte) 4); - out.writeFloat((Float) sortValue); - } else if (type == Double.class) { - out.writeByte((byte) 5); - out.writeDouble((Double) sortValue); - } else if (type == Byte.class) { - out.writeByte((byte) 6); - out.writeByte((Byte) sortValue); - } else if (type == Short.class) { - out.writeByte((byte) 7); - out.writeShort((Short) sortValue); - } else if (type == Boolean.class) { - out.writeByte((byte) 8); - out.writeBoolean((Boolean) sortValue); - } else { - throw new IOException("Can't handle sort field value of type [" + type + "]"); - } - } + out.writeArray(Lucene::writeSortValue, this.formattedSortValues); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeArray(Lucene::writeSortValue, this.rawSortValues); } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (sortValues.length > 0) { + if (formattedSortValues.length > 0) { builder.startArray(Fields.SORT); - for (Object sortValue : sortValues) { + for (Object sortValue : formattedSortValues) { builder.value(sortValue); } builder.endArray(); @@ -142,24 +99,37 @@ public static SearchSortValues fromXContent(XContentParser parser) throws IOExce return new SearchSortValues(parser.list().toArray()); } - public Object[] sortValues() { - return sortValues; + /** + * Returns the formatted version of the values that sorting was performed against + */ + public Object[] getFormattedSortValues() { + return formattedSortValues; + } + + /** + * Returns the raw version of the values that sorting was performed against + */ + public Object[] getRawSortValues() { + return rawSortValues; } @Override - public boolean equals(Object obj) { - if (this == obj) { + public boolean equals(Object o) { + if (this == o) { return true; } - if (obj == null || getClass() != obj.getClass()) { + if (o == null || getClass() != o.getClass()) { return false; } - SearchSortValues other = (SearchSortValues) obj; - return Arrays.equals(sortValues, other.sortValues); + SearchSortValues that = (SearchSortValues) o; + return Arrays.equals(formattedSortValues, that.formattedSortValues) && + Arrays.equals(rawSortValues, that.rawSortValues); } @Override public int hashCode() { - return Arrays.hashCode(sortValues); + int result = Arrays.hashCode(formattedSortValues); + result = 31 * result + Arrays.hashCode(rawSortValues); + return result; } } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index ea894a2edd09a..1891be362b814 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -531,24 +531,26 @@ public void testSortValueSerialization() throws IOException { } public static Object randomSortValue() { - switch(randomIntBetween(0, 8)) { + switch(randomIntBetween(0, 9)) { case 0: - return randomAlphaOfLengthBetween(3, 10); + return null; case 1: - return randomInt(); + return randomAlphaOfLengthBetween(3, 10); case 2: - return randomLong(); + return randomInt(); case 3: - return randomFloat(); + return randomLong(); case 4: - return randomDouble(); + return randomFloat(); case 5: - return randomByte(); + return randomDouble(); case 6: - return randomShort(); + return randomByte(); case 7: - return randomBoolean(); + return randomShort(); case 8: + return randomBoolean(); + case 9: return new BytesRef(randomAlphaOfLengthBetween(3, 10)); default: throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java index f6b8dc828f4e6..797b5dd888ed4 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.LuceneTests; import org.elasticsearch.common.xcontent.ToXContent; @@ -31,23 +32,36 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.RandomObjects; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.Arrays; +import java.util.Base64; public class SearchSortValuesTests extends AbstractSerializingTestCase { public static SearchSortValues createTestItem(XContentType xContentType, boolean transportSerialization) { int size = randomIntBetween(1, 20); Object[] values = new Object[size]; - DocValueFormat[] sortValueFormats = new DocValueFormat[size]; - for (int i = 0; i < size; i++) { - Object sortValue = randomSortValue(xContentType, transportSerialization); - values[i] = sortValue; - //make sure that for BytesRef, we provide a specific doc value format that overrides format(BytesRef) - sortValueFormats[i] = sortValue instanceof BytesRef ? DocValueFormat.RAW : randomDocValueFormat(); + if (transportSerialization) { + DocValueFormat[] sortValueFormats = new DocValueFormat[size]; + for (int i = 0; i < size; i++) { + Object sortValue = randomSortValue(xContentType, transportSerialization); + values[i] = sortValue; + //make sure that for BytesRef, we provide a specific doc value format that overrides format(BytesRef) + sortValueFormats[i] = sortValue instanceof BytesRef ? DocValueFormat.RAW : randomDocValueFormat(); + } + return new SearchSortValues(values, sortValueFormats); + } else { + //xcontent serialization doesn't write/parse the raw sort values, only the formatted ones + for (int i = 0; i < size; i++) { + Object sortValue = randomSortValue(xContentType, transportSerialization); + //make sure that BytesRef are not provided as formatted values + sortValue = sortValue instanceof BytesRef ? DocValueFormat.RAW.format((BytesRef)sortValue) : sortValue; + values[i] = sortValue; + } + return new SearchSortValues(values); } - return new SearchSortValues(values, sortValueFormats); } private static Object randomSortValue(XContentType xContentType, boolean transportSerialization) { @@ -79,7 +93,7 @@ protected SearchSortValues createXContextTestInstance(XContentType xContentType) @Override protected SearchSortValues createTestInstance() { - return createTestItem(randomFrom(XContentType.values()), true); + return createTestItem(randomFrom(XContentType.values()), randomBoolean()); } @Override @@ -113,20 +127,32 @@ public void testToXContent() throws IOException { @Override protected SearchSortValues mutateInstance(SearchSortValues instance) { - Object[] sortValues = instance.sortValues(); - if (sortValues.length == 0) { - return createTestInstance(); - } + Object[] sortValues = instance.getFormattedSortValues(); if (randomBoolean()) { return new SearchSortValues(new Object[0]); } Object[] values = Arrays.copyOf(sortValues, sortValues.length + 1); - values[sortValues.length] = randomSortValue(randomFrom(XContentType.values()), true); + values[sortValues.length] = randomSortValue(randomFrom(XContentType.values()), randomBoolean()); return new SearchSortValues(values); } - @Override - protected SearchSortValues copyInstance(SearchSortValues instance, Version version) { - return new SearchSortValues(Arrays.copyOf(instance.sortValues(), instance.sortValues().length)); + //TODO rename and update version after backport + public void testSerializationPre70() throws IOException { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); + SearchSortValues original = createTestInstance(); + SearchSortValues deserialized = copyInstance(original, version); + assertArrayEquals(original.getFormattedSortValues(), deserialized.getFormattedSortValues()); + assertEquals(0, deserialized.getRawSortValues().length); + } + + //TODO rename method and adapt versions after backport + public void testReadFromPre70() throws IOException { + try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode("AwIAAAABAQEyBUAIAAAAAAAAAAAAAAAA"))) { + in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0))); + SearchSortValues deserialized = new SearchSortValues(in); + SearchSortValues expected = new SearchSortValues(new Object[]{1, "2", 3d}); + assertEquals(expected, deserialized); + assertEquals(0, deserialized.getRawSortValues().length); + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java index 3478c14cfda53..cb7f5ff4a229e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java @@ -27,6 +27,6 @@ public abstract class AbstractWireSerializingTestCase exten @Override protected T copyInstance(T instance, Version version) throws IOException { - return copyWriteable(instance, getNamedWriteableRegistry(), instanceReader()); + return copyWriteable(instance, getNamedWriteableRegistry(), instanceReader(), version); } } From 997703f528126fec32a159c261c4fc903d506184 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Tue, 18 Dec 2018 10:11:39 +0100 Subject: [PATCH 53/57] Watcher: Ensure all internal search requests count hits (#36697) In previous commits only the stored toXContent version of a search request was using the old format. However an executed search request was already disabling hit counts. In 7.0 hit counts will stay enabled by default to allow for proper migration. Closes #36177 --- .../put_watch/91_search_total_hits_as_int.yml | 115 ++++++++++++++++++ .../search/WatcherSearchTemplateRequest.java | 5 +- .../WatcherSearchTemplateRequestTests.java | 24 +++- .../AbstractWatcherIntegrationTestCase.java | 2 +- .../test/integration/BasicWatcherTests.java | 18 +-- .../test/integration/WatchAckTests.java | 6 +- 6 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/91_search_total_hits_as_int.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/91_search_total_hits_as_int.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/91_search_total_hits_as_int.yml new file mode 100644 index 0000000000000..46986438ee4a4 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/91_search_total_hits_as_int.yml @@ -0,0 +1,115 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + + - do: + index: + index: my_test_index + type: doc + id: my_id + refresh: true + body: > + { + "key": "value" + } + +--- +"Test search input includes hits by default": + + - do: + xpack.watcher.execute_watch: + body: > + { + "watch" : { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "search" : { + "request" : { + "indices" : [ "my_test_index" ], + "body" : { + "query": { + "match_all" : {} + } + } + } + } + }, + "condition": { + "compare": { + "ctx.payload.hits.total": { + "gt": 0 + } + } + }, + "actions": { + "logging" : { + "logging" : { + "text" : "Logging from a test" + } + } + } + } + } + + - match: { watch_record.result.condition.met: true } + + +--- +"Test search transform includes hits by default": + + - do: + xpack.watcher.execute_watch: + body: > + { + "watch" : { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "simple": { + "foo": "bar" + } + }, + "transform" : { + "search" : { + "request" : { + "indices" : [ "my_test_index" ], + "body" : { + "query": { + "match_all" : {} + } + } + } + } + }, + "actions": { + "indexme" : { + "condition": { + "compare": { + "ctx.payload.hits.total": { + "gt": 0 + } + } + }, + "index" : { + "index" : "my_test_index", + "doc_type" : "doc", + "doc_id": "my-id" + } + } + } + } + } + + - do: + get: + index: my_test_index + type: doc + id: my_id + + - match: { _source.key: "value" } + diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java index 8619143c83d38..245093f3b385c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java @@ -41,7 +41,7 @@ public class WatcherSearchTemplateRequest implements ToXContentObject { private final IndicesOptions indicesOptions; private final Script template; private final BytesReference searchSource; - private boolean restTotalHitsAsInt; + private boolean restTotalHitsAsInt = true; public WatcherSearchTemplateRequest(String[] indices, String[] types, SearchType searchType, IndicesOptions indicesOptions, BytesReference searchSource) { @@ -184,7 +184,8 @@ public static WatcherSearchTemplateRequest fromXContent(XContentParser parser, S IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; BytesReference searchSource = null; Script template = null; - boolean totalHitsAsInt = false; + // TODO this is to retain BWC compatibility in 7.0 and can be removed for 8.0 + boolean totalHitsAsInt = true; XContentParser.Token token; String currentFieldName = null; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequestTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequestTests.java index b6111cb97e2b7..07fd3c7765485 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequestTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequestTests.java @@ -15,6 +15,7 @@ import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class WatcherSearchTemplateRequestTests extends ESTestCase { @@ -28,7 +29,26 @@ public void testFromXContentWithTemplateCustomLang() throws IOException { assertTemplate(source, "custom-script", "painful", singletonMap("bar", "baz")); } - private void assertTemplate(String source, String expectedScript, String expectedLang, Map expectedParams) { + public void testDefaultHitCountsDefaults() throws IOException { + assertHitCount("{}", true); + } + + public void testDefaultHitCountsConfigured() throws IOException { + boolean hitCountsAsInt = randomBoolean(); + String source = "{ \"rest_total_hits_as_int\" : " + hitCountsAsInt + " }"; + assertHitCount(source, hitCountsAsInt); + } + + private void assertHitCount(String source, boolean expectedHitCountAsInt) throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { + parser.nextToken(); + WatcherSearchTemplateRequest request = WatcherSearchTemplateRequest.fromXContent(parser, SearchType.QUERY_THEN_FETCH); + assertThat(request.isRestTotalHitsAsint(), is(expectedHitCountAsInt)); + } + } + + private void assertTemplate(String source, String expectedScript, String expectedLang, Map expectedParams) + throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { parser.nextToken(); WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, randomFrom(SearchType.values())); @@ -36,8 +56,6 @@ private void assertTemplate(String source, String expectedScript, String expecte assertThat(result.getTemplate().getIdOrCode(), equalTo(expectedScript)); assertThat(result.getTemplate().getLang(), equalTo(expectedLang)); assertThat(result.getTemplate().getParams(), equalTo(expectedParams)); - } catch (IOException e) { - fail("Failed to parse watch search request: " + e.getMessage()); } } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index b4a05ef5a3439..615fc8fbd08f9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -340,7 +340,7 @@ protected void assertWatchWithMinimumPerformedActionsCount(final String watchNam assertThat("could not find executed watch record for watch " + watchName, searchResponse.getHits().getTotalHits().value, greaterThanOrEqualTo(minimumExpectedWatchActionsWithActionPerformed)); if (assertConditionMet) { - assertThat((Integer) XContentMapValues.extractValue("result.input.payload.hits.total.value", + assertThat((Integer) XContentMapValues.extractValue("result.input.payload.hits.total", searchResponse.getHits().getAt(0).getSourceAsMap()), greaterThanOrEqualTo(1)); } }); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java index a037fba6c4db8..2b7d5706d233e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java @@ -75,7 +75,7 @@ public void testIndexWatch() throws Exception { .setSource(watchBuilder() .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS))) .input(searchInput(request)) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L)) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)) .addAction("_logger", loggingAction("_logging") .setCategory("_category"))) .get(); @@ -95,7 +95,7 @@ public void testIndexWatchRegisterWatchBeforeTargetIndex() throws Exception { .setSource(watchBuilder() .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS))) .input(searchInput(searchRequest)) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L))) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))) .get(); timeWarp().trigger("_name"); // The watch's condition won't meet because there is no data that matches with the query @@ -119,7 +119,7 @@ public void testDeleteWatch() throws Exception { .setSource(watchBuilder() .trigger(schedule(cron("0/1 * * * * ? 2020"))) .input(searchInput(searchRequest)) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L))) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))) .get(); assertThat(indexResponse.isCreated(), is(true)); DeleteWatchResponse deleteWatchResponse = watcherClient.prepareDeleteWatch("_name").get(); @@ -180,7 +180,7 @@ public void testModifyWatches() throws Exception { .addAction("_id", indexAction("idx", "action")); watcherClient().preparePutWatch("_name") - .setSource(source.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L))) + .setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))) .get(); timeWarp().clock().fastForwardSeconds(5); @@ -188,7 +188,7 @@ public void testModifyWatches() throws Exception { assertWatchWithMinimumPerformedActionsCount("_name", 0, false); watcherClient().preparePutWatch("_name") - .setSource(source.condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 0L))) + .setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L))) .get(); timeWarp().clock().fastForwardSeconds(5); @@ -199,7 +199,7 @@ public void testModifyWatches() throws Exception { watcherClient().preparePutWatch("_name") .setSource(source .trigger(schedule(Schedules.cron("0/1 * * * * ? 2020"))) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 0L))) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L))) .get(); timeWarp().clock().fastForwardSeconds(5); @@ -245,7 +245,7 @@ public void testInputFiltering() throws Exception { .setSource(watchBuilder() .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS))) .input(searchInput(request).extractKeys("hits.total.value")) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.EQ, 1L))) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))) .get(); // in this watcher the condition will fail, because max_score isn't extracted, only total: watcherClient.preparePutWatch("_name2") @@ -265,7 +265,7 @@ public void testInputFiltering() throws Exception { SearchResponse searchResponse = searchWatchRecords(builder -> builder.setQuery(matchQuery("watch_id", "_name1"))); assertHitCount(searchResponse, 1); XContentSource source = xContentSource(searchResponse.getHits().getAt(0).getSourceRef()); - assertThat(source.getValue("result.input.payload.hits.total.value"), equalTo((Object) 1)); + assertThat(source.getValue("result.input.payload.hits.total"), equalTo((Object) 1)); } public void testPutWatchWithNegativeSchedule() throws Exception { @@ -349,7 +349,7 @@ private void testConditionSearch(WatcherSearchTemplateRequest request) throws Ex .setSource(watchBuilder() .trigger(schedule(interval("5s"))) .input(searchInput(request)) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GTE, 3L))) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3L))) .get(); logger.info("created watch [{}] at [{}]", watchName, new DateTime(Clock.systemUTC().millis())); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java index 805bda6f9d593..cb8fc9edb6f14 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java @@ -67,7 +67,7 @@ public void testAckSingleAction() throws Exception { .setSource(watchBuilder() .trigger(schedule(cron("0/5 * * * * ? *"))) .input(searchInput(templateRequest(searchSource(), "events"))) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L)) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L)) .transform(searchTransform(templateRequest(searchSource(), "events"))) .addAction("_a1", indexAction("actions1", "doc")) .addAction("_a2", indexAction("actions2", "doc")) @@ -127,7 +127,7 @@ public void testAckAllActions() throws Exception { .setSource(watchBuilder() .trigger(schedule(cron("0/5 * * * * ? *"))) .input(searchInput(templateRequest(searchSource(), "events"))) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L)) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L)) .transform(searchTransform(templateRequest(searchSource(), "events"))) .addAction("_a1", indexAction("actions1", "doc")) .addAction("_a2", indexAction("actions2", "doc")) @@ -195,7 +195,7 @@ public void testAckWithRestart() throws Exception { .setSource(watchBuilder() .trigger(schedule(cron("0/5 * * * * ? *"))) .input(searchInput(templateRequest(searchSource(), "events"))) - .condition(new CompareCondition("ctx.payload.hits.total.value", CompareCondition.Op.GT, 0L)) + .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L)) .transform(searchTransform(templateRequest(searchSource(), "events"))) .addAction("_id", indexAction("actions", "action"))) .get(); From 17f27cc32a246b971c53f3f8d87c25df623ba314 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 18 Dec 2018 10:43:14 +0100 Subject: [PATCH 54/57] Ensure MapperService#getAllMetaFields elements order is deterministic (#36739) MapperService#getAllMetaFields returns an array, which is created out of an `ObjectHashSet`. Such set does not guarantee deterministic hash ordering. The array returned by its toArray may be sorted differently at each run. This caused some repeatability issues in our tests (see #29080) as we pick random fields from the array of possible metadata fields, but that won't be repeatable if the input array is sorted differently at every run. Once setting the tests seed, hppc picks that up and the sorting is deterministic, but failures don't repeat with the seed that gets printed out originally (as a seed was not originally set). See also https://issues.carrot2.org/projects/HPPC/issues/HPPC-173. With this commit, we simply create a static sorted array that is used for `getAllMetaFields`. The change is in production code but really affects only testing as the only production usage of this method was to iterate through all values when parsing fields in the high-level REST client code. Anyways, this seems like a good change as returning an array would imply that it's deterministically sorted. --- .../elasticsearch/index/mapper/MapperService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 7663ec817a0db..6424e75eaf662 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -109,10 +109,11 @@ public enum MergeReason { //TODO this needs to be cleaned up: _timestamp and _ttl are not supported anymore, _field_names, _seq_no, _version and _source are //also missing, not sure if on purpose. See IndicesModule#getMetadataMappers - private static ObjectHashSet META_FIELDS = ObjectHashSet.from( - "_id", "_type", "_routing", "_index", - "_size", "_timestamp", "_ttl", IgnoredFieldMapper.NAME - ); + private static final String[] SORTED_META_FIELDS = new String[]{ + "_id", IgnoredFieldMapper.NAME, "_index", "_routing", "_size", "_timestamp", "_ttl", "_type" + }; + + private static final ObjectHashSet META_FIELDS = ObjectHashSet.from(SORTED_META_FIELDS); private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(MapperService.class)); @@ -762,7 +763,7 @@ public static boolean isMetadataField(String fieldName) { } public static String[] getAllMetaFields() { - return META_FIELDS.toArray(String.class); + return Arrays.copyOf(SORTED_META_FIELDS, SORTED_META_FIELDS.length); } /** An analyzer wrapper that can lookup fields within the index mappings */ @@ -789,5 +790,4 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { return defaultAnalyzer; } } - } From 557a5be62aec7afcdcac22098bf5a099cefe0a5f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 18 Dec 2018 10:43:11 +0100 Subject: [PATCH 55/57] [TEST] Ensure shard follow tasks have really stopped. Relates to #36696 --- .../org/elasticsearch/xpack/CcrSingleNodeTestCase.java | 9 ++++++--- .../elasticsearch/xpack/ccr/action/FollowStatsIT.java | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index 417de7cd985c5..5311f54762be2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -58,10 +58,13 @@ public void setupLocalRemote() { } @After - public void remoteLocalRemote() throws Exception { + public void purgeCCRMetadata() throws Exception { ClusterService clusterService = getInstanceFromNode(ClusterService.class); removeCCRRelatedMetadataFromClusterState(clusterService); + } + @After + public void removeLocalRemote() { ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); updateSettingsRequest.transientSettings(Settings.builder().put("cluster.remote.local.seeds", (String) null)); assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); @@ -70,8 +73,8 @@ public void remoteLocalRemote() throws Exception { protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex) { ResumeFollowAction.Request request = new ResumeFollowAction.Request(); request.setFollowerIndex(followerIndex); - request.setMaxRetryDelay(TimeValue.timeValueMillis(10)); - request.setReadPollTimeout(TimeValue.timeValueMillis(10)); + request.setMaxRetryDelay(TimeValue.timeValueMillis(1)); + request.setReadPollTimeout(TimeValue.timeValueMillis(1)); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowStatsIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowStatsIT.java index 42901e2697167..bf6f080099088 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowStatsIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowStatsIT.java @@ -13,11 +13,13 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.xpack.CcrSingleNodeTestCase; +import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction; import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import java.util.Comparator; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -106,6 +108,12 @@ public void testFollowStatsApiFollowerIndexFiltering() throws Exception { assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet()); + + assertBusy(() -> { + List responseList = + client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet().getFollowStats().getStatsResponses(); + assertThat(responseList.size(), equalTo(0)); + }); } } From 7872365e95f0fcb25d0d9ababe3472accd16598e Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 18 Dec 2018 10:56:02 +0100 Subject: [PATCH 56/57] Expose Sequence Number based Optimistic Concurrency Control in the rest layer (#36721) Relates #36148 Relates #10708 --- .../resources/rest-api-spec/api/delete.json | 8 +++ .../resources/rest-api-spec/api/index.json | 8 +++ .../rest-api-spec/test/index/30_cas.yml | 50 +++++++++++++++++++ .../elasticsearch/action/get/GetResponse.java | 2 +- .../elasticsearch/index/get/GetResult.java | 2 +- .../action/document/RestDeleteAction.java | 4 ++ .../rest/action/document/RestIndexAction.java | 4 ++ 7 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/30_cas.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json index bbe30d3a8484c..2e75465bf601e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json @@ -43,6 +43,14 @@ "type" : "time", "description" : "Explicit operation timeout" }, + "if_seq_no_match" : { + "type" : "number", + "description" : "only perform the delete operation if the last operation that has changed the document has the specified sequence number" + }, + "if_primary_term_match" : { + "type" : "number", + "description" : "only perform the delete operation if the last operation that has changed the document has the specified primary term" + }, "version" : { "type" : "number", "description" : "Explicit version number for concurrency control" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json index 574206a0dc3ed..155707bbdcf14 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json @@ -57,6 +57,14 @@ "options" : ["internal", "external", "external_gte", "force"], "description" : "Specific version type" }, + "if_seq_no_match" : { + "type" : "number", + "description" : "only perform the index operation if the last operation that has changed the document has the specified sequence number" + }, + "if_primary_term_match" : { + "type" : "number", + "description" : "only perform the index operation if the last operation that has changed the document has the specified primary term" + }, "pipeline" : { "type" : "string", "description" : "The pipeline id to preprocess incoming documents with" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_cas.yml new file mode 100644 index 0000000000000..b8c60e5a7cf8b --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_cas.yml @@ -0,0 +1,50 @@ +--- +"Compare And Swap Sequence Numbers": + + - skip: + version: " - 6.99.99" + reason: cas ops are introduced in 7.0.0 + + - do: + index: + index: test_1 + id: 1 + body: { foo: bar } + - match: { _version: 1} + - set: { _seq_no: seqno } + - set: { _primary_term: primary_term } + + - do: + get: + index: test_1 + id: 1 + - match: { _seq_no: $seqno } + - match: { _primary_term: $primary_term } + + - do: + catch: conflict + index: + index: test_1 + id: 1 + if_seq_no_match: 10000 + if_primary_term_match: $primary_term + body: { foo: bar2 } + + - do: + catch: conflict + index: + index: test_1 + id: 1 + if_seq_no_match: $seqno + if_primary_term_match: 1000 + body: { foo: bar2 } + + - do: + index: + index: test_1 + id: 1 + if_seq_no_match: $seqno + if_primary_term_match: $primary_term + body: { foo: bar2 } + + - match: { _version: 2 } diff --git a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java index fbcb47b5fad36..b9383785678b7 100644 --- a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java @@ -91,7 +91,7 @@ public long getVersion() { } /** - * The sequence number assigned to the last operation to have changed this document, if found. + * The sequence number assigned to the last operation that has changed this document, if found. */ public long getSeqNo() { return getResult.getSeqNo(); diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index b98d766dd4e3f..2b3b1b8f4f231 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -131,7 +131,7 @@ public long getVersion() { } /** - * The sequence number assigned to the last operation to have changed this document, if found. + * The sequence number assigned to the last operation that has changed this document, if found. */ public long getSeqNo() { return seqNo; diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java index 87cc7a0fb41a4..1891b29d175c9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java @@ -66,6 +66,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC deleteRequest.setRefreshPolicy(request.param("refresh")); deleteRequest.version(RestActions.parseVersion(request)); deleteRequest.versionType(VersionType.fromString(request.param("version_type"), deleteRequest.versionType())); + deleteRequest.setIfMatch( + request.paramAsLong("if_seq_no_match", deleteRequest.ifSeqNoMatch()), + request.paramAsLong("if_primary_term_match", deleteRequest.ifPrimaryTermMatch()) + ); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 619fd811e6a7c..2a072560272bf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -93,6 +93,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); + indexRequest.ifMatch( + request.paramAsLong("if_seq_no_match", indexRequest.ifSeqNoMatch()), + request.paramAsLong("if_primary_term_match", indexRequest.ifPrimaryTermMatch()) + ); String sOpType = request.param("op_type"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { From 28998f4fc846c8d1c6d5b2fd5a74f167017c3005 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 18 Dec 2018 10:57:14 +0000 Subject: [PATCH 57/57] [ML] Mute MlDistributedFailureIT --- .../xpack/ml/integration/MlDistributedFailureIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index c1c0d572c04f4..50092a766e7d9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -72,6 +72,7 @@ public void testFailOver() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32905") public void testLoseDedicatedMasterNode() throws Exception { internalCluster().ensureAtMostNumDataNodes(0); logger.info("Starting dedicated master node...");