diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java index 6d6e5ade8275a..c5a544eb7854a 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugin.noop.NoopPlugin; import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction; @@ -80,7 +81,7 @@ public TransportBulkRequestExecutor(TransportClient client, String indexName, St public boolean bulkIndex(List bulkData) { NoopBulkRequestBuilder builder = NoopBulkAction.INSTANCE.newRequestBuilder(client); for (String bulkItem : bulkData) { - builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8))); + builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } BulkResponse bulkResponse; try { diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java index ceaf9f8cc9d17..1034e722e8789 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java @@ -33,6 +33,7 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; public class NoopBulkRequestBuilder extends ActionRequestBuilder implements WriteRequestBuilder { @@ -95,17 +96,17 @@ public NoopBulkRequestBuilder add(UpdateRequestBuilder request) { /** * Adds a framed data in binary format */ - public NoopBulkRequestBuilder add(byte[] data, int from, int length) throws Exception { - request.add(data, from, length, null, null); + public NoopBulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception { + request.add(data, from, length, null, null, xContentType); return this; } /** * Adds a framed data in binary format */ - public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) - throws Exception { - request.add(data, from, length, defaultIndex, defaultType); + public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws Exception { + request.add(data, from, length, defaultIndex, defaultType, xContentType); return this; } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java index 7765b1053443e..5d4215b3ded4c 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -73,7 +73,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); - bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true); + bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true, + request.getXContentType()); // short circuit the call to the transport layer return channel -> { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index a06175a598bf2..f0f8d50b4c1c9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -142,14 +142,28 @@ public PutRepositoryRequest settings(Settings.Builder settings) { /** * Sets the repository settings. * - * @param source repository settings in json, yaml or properties format + * @param source repository settings in json or yaml format * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection */ + @Deprecated public PutRepositoryRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the repository settings. + * + * @param source repository settings in json or yaml format + * @param xContentType the content type of the source + * @return this request + */ + public PutRepositoryRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the repository settings. * @@ -160,7 +174,7 @@ public PutRepositoryRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 39cfa6af7f750..aed09daff2a86 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -89,16 +90,30 @@ public PutRepositoryRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets the repository settings in Json, Yaml or properties format + * Sets the repository settings in Json or Yaml format * * @param source repository settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection */ + @Deprecated public PutRepositoryRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets the repository settings in Json or Yaml format + * + * @param source repository settings + * @param xContentType the contenty type of the source + * @return this builder + */ + public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets the repository settings * diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index e5f5bdb7fb512..bd0110e644cb6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -83,12 +83,22 @@ public ClusterUpdateSettingsRequest transientSettings(Settings.Builder settings) /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest transientSettings(String source) { this.transientSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) { + this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ @@ -97,7 +107,7 @@ public ClusterUpdateSettingsRequest transientSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - transientSettings(builder.string()); + transientSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -122,12 +132,22 @@ public ClusterUpdateSettingsRequest persistentSettings(Settings.Builder settings /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest persistentSettings(String source) { this.persistentSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) { + this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ @@ -136,7 +156,7 @@ public ClusterUpdateSettingsRequest persistentSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - persistentSettings(builder.string()); + persistentSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index f0492edfeb19f..906b1867b1f09 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -52,12 +53,22 @@ public ClusterUpdateSettingsRequestBuilder setTransientSettings(Settings.Builder /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) { request.transientSettings(settings); return this; } + /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) { + request.transientSettings(settings, xContentType); + return this; + } + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ @@ -84,12 +95,22 @@ public ClusterUpdateSettingsRequestBuilder setPersistentSettings(Settings.Builde /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) { request.persistentSettings(settings); return this; } + /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) { + request.persistentSettings(settings, xContentType); + return this; + } + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index ae715050e80b5..3267b6d9c96ed 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -288,18 +288,34 @@ public CreateSnapshotRequest settings(Settings.Builder settings) { } /** - * Sets repository-specific snapshot settings in JSON, YAML or properties format + * Sets repository-specific snapshot settings in JSON or YAML format *

* See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets repository-specific snapshot settings in JSON or YAML format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public CreateSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets repository-specific snapshot settings. *

@@ -312,7 +328,7 @@ public CreateSnapshotRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index ebdd206b5c3e8..d3b5e12351c28 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -147,12 +148,28 @@ public CreateSnapshotRequestBuilder setSettings(Settings.Builder settings) { * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets repository-specific snapshot settings in YAML or JSON format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets repository-specific snapshot settings. *

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index a7bbd02ee54eb..9d8ed49aaa09f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -313,18 +313,34 @@ public RestoreSnapshotRequest settings(Settings.Builder settings) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets repository-specific restore settings in JSON or YAML format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public RestoreSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets repository-specific restore settings *

@@ -337,7 +353,7 @@ public RestoreSnapshotRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -436,12 +452,22 @@ public RestoreSnapshotRequest indexSettings(Settings.Builder settings) { /** * Sets settings that should be added/changed in all restored indices + * @deprecated use {@link #indexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest indexSettings(String source) { this.indexSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets settings that should be added/changed in all restored indices + */ + public RestoreSnapshotRequest indexSettings(String source, XContentType xContentType) { + this.indexSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets settings that should be added/changed in all restored indices */ @@ -449,7 +475,7 @@ public RestoreSnapshotRequest indexSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - indexSettings(builder.string()); + indexSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 661a1a1d018af..807e238724364 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.List; import java.util.Map; @@ -153,18 +154,34 @@ public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets repository-specific restore settings in JSON or YAML format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets repository-specific restore settings *

@@ -251,12 +268,26 @@ public RestoreSnapshotRequestBuilder setIndexSettings(Settings.Builder settings) * * @param source index settings * @return this builder + * @deprecated use {@link #setIndexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setIndexSettings(String source) { request.indexSettings(source); return this; } + /** + * Sets index settings that should be added or replaced during restore + * + * @param source index settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setIndexSettings(String source, XContentType xContentType) { + request.indexSettings(source, xContentType); + return this; + } + /** * Sets index settings that should be added or replaced during restore * diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 0dcfe22514233..28c70dc45b856 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -19,14 +19,18 @@ package org.elasticsearch.action.admin.cluster.storedscripts; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -35,17 +39,23 @@ public class PutStoredScriptRequest extends AcknowledgedRequest { @@ -32,19 +33,29 @@ public PutStoredScriptRequestBuilder(ElasticsearchClient client, PutStoredScript public PutStoredScriptRequestBuilder setId(String id) { request.id(id); - return this; } - public PutStoredScriptRequestBuilder setLang(String lang) { - request.lang(lang); - + /** + * Set the source of the script. + * @deprecated this method requires content type detection. Use {@link #setContent(BytesReference, XContentType)} instead + */ + @Deprecated + public PutStoredScriptRequestBuilder setContent(BytesReference content) { + request.content(content); return this; } - public PutStoredScriptRequestBuilder setContent(BytesReference content) { - request.content(content); + /** + * Set the source of the script along with the content type of the source + */ + public PutStoredScriptRequestBuilder setContent(BytesReference source, XContentType xContentType) { + request.content(source, xContentType); + return this; + } + public PutStoredScriptRequestBuilder setLang(String lang) { + request.lang(lang); return this; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 6d0824eeb31c5..08f220e0199d8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -75,7 +75,7 @@ public static class NameOrDefinition implements Writeable { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(definition); - this.definition = Settings.builder().loadFromSource(builder.string()).build(); + this.definition = Settings.builder().loadFromSource(builder.string(), builder.contentType()).build(); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse [" + definition + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 203483d89b3f4..798c08bd180d0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -43,10 +44,11 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -169,19 +171,29 @@ public CreateIndexRequest settings(Settings.Builder settings) { } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #source(String, XContentType)} instead to avoid content type detection */ + @Deprecated public CreateIndexRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Allows to set the settings using a json builder. */ public CreateIndexRequest settings(XContentBuilder builder) { try { - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate json settings from builder", e); } @@ -196,7 +208,7 @@ public CreateIndexRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -208,13 +220,42 @@ public CreateIndexRequest settings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, String, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequest mapping(String type, String source) { + return mapping(type, new BytesArray(source), XContentFactory.xContentType(source)); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequest mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the content type of the mapping source + */ + private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) { if (mappings.containsKey(type)) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } - mappings.put(type, source); - return this; + Objects.requireNonNull(xContentType); + try { + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert to json", e); + } } /** @@ -232,15 +273,7 @@ public CreateIndexRequest cause(String cause) { * @param source The mapping source */ public CreateIndexRequest mapping(String type, XContentBuilder source) { - if (mappings.containsKey(type)) { - throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); - } - try { - mappings.put(type, source.string()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); - } - return this; + return mapping(type, source.bytes(), source.contentType()); } /** @@ -261,7 +294,7 @@ public CreateIndexRequest mapping(String type, Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -331,9 +364,18 @@ public CreateIndexRequest alias(Alias alias) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public CreateIndexRequest source(String source) { - return source(source.getBytes(StandardCharsets.UTF_8)); + return source(new BytesArray(source)); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(String source, XContentType xContentType) { + return source(new BytesArray(source), xContentType); } /** @@ -345,7 +387,9 @@ public CreateIndexRequest source(XContentBuilder source) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], XContentType)} */ + @Deprecated public CreateIndexRequest source(byte[] source) { return source(source, 0, source.length); } @@ -353,6 +397,15 @@ public CreateIndexRequest source(byte[] source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -360,13 +413,27 @@ public CreateIndexRequest source(byte[] source, int offset, int length) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequest source(BytesReference source) { XContentType xContentType = XContentFactory.xContentType(source); - if (xContentType != null) { - source(XContentHelper.convertToMap(source, false).v2()); - } else { - settings(source.utf8ToString()); - } + source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); + source(XContentHelper.convertToMap(source, false, xContentType).v2()); return this; } @@ -483,7 +550,13 @@ public void readFrom(StreamInput in) throws IOException { readTimeout(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + final String type = in.readString(); + String source = in.readString(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport + // we do not know the content type that comes from earlier versions so we autodetect and convert + source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); + } + mappings.put(type, source); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index eaae4d53b73fd..237c88244b4cd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -76,13 +77,23 @@ public CreateIndexRequestBuilder setSettings(XContentBuilder builder) { } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * A simplified version of settings that takes key value pairs settings. */ @@ -104,12 +115,26 @@ public CreateIndexRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} to avoid content type auto-detection */ + @Deprecated public CreateIndexRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + /** * The cause for this index creation. */ @@ -191,7 +216,9 @@ public CreateIndexRequestBuilder addAlias(Alias alias) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public CreateIndexRequestBuilder setSource(String source) { request.source(source); return this; @@ -200,6 +227,16 @@ public CreateIndexRequestBuilder setSource(String source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; @@ -208,6 +245,16 @@ public CreateIndexRequestBuilder setSource(BytesReference source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source) { request.source(source); return this; @@ -216,11 +263,29 @@ public CreateIndexRequestBuilder setSource(byte[] source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; } + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + /** * Sets the settings and mappings as a single source. */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index e0cedcf841e47..3f4ddaf08db2c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; import java.io.IOException; @@ -108,20 +109,25 @@ public String fullName() { /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */ public Map sourceAsMap() { - return XContentHelper.convertToMap(source, true).v2(); + return XContentHelper.convertToMap(source, true, XContentType.JSON).v2(); } public boolean isNull() { return NULL.fullName().equals(fullName) && NULL.source.length() == source.length(); } + //pkg-private for testing + BytesReference getSource() { + return source; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("full_name", fullName); if (params.paramAsBoolean("pretty", false)) { builder.field("mapping", sourceAsMap()); } else { - builder.rawField("mapping", source); + builder.rawField("mapping", source, XContentType.JSON); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 864c6703c48e0..ec84ce4e34779 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -220,4 +220,4 @@ private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuildertrue iff the settings update should only add but not update settings. If the setting already exists * it should not be overwritten by this update. The default is false @@ -146,14 +156,14 @@ public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) { } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) */ @SuppressWarnings("unchecked") public UpdateSettingsRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java index 36dfbf3b2d49b..a9cecbfc5a434 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -70,15 +71,25 @@ public UpdateSettingsRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public UpdateSettingsRequestBuilder setSettings(String source) { request.settings(source); return this; } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + */ + public UpdateSettingsRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** + * Sets the settings to be updated */ public UpdateSettingsRequestBuilder setSettings(Map source) { request.settings(source); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index a7d6241d31e2b..b0c13540dfa0b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -45,11 +45,13 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -179,21 +181,31 @@ public PutIndexTemplateRequest settings(Settings.Builder settings) { } /** - * The settings to create the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml format). + * @deprecated use {@link #settings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** - * The settings to crete the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml format). + */ + public PutIndexTemplateRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * The settings to create the index template with (either json or yaml format). */ public PutIndexTemplateRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -209,10 +221,23 @@ public Settings settings() { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest mapping(String type, String source) { - mappings.put(type, source); - return this; + XContentType xContentType = XContentFactory.xContentType(source); + return mapping(type, source, xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); } /** @@ -234,12 +259,24 @@ public String cause() { * @param source The mapping source */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { + return mapping(type, source.bytes(), source.contentType()); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); try { - mappings.put(type, source.string()); + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); + throw new UncheckedIOException("failed to convert source to json", e); } - return this; } /** @@ -256,7 +293,7 @@ public PutIndexTemplateRequest mapping(String type, Map source) try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -280,7 +317,7 @@ public Map mappings() { */ public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { try { - return source(templateBuilder.bytes()); + return source(templateBuilder.bytes(), templateBuilder.contentType()); } catch (Exception e) { throw new IllegalArgumentException("Failed to build json for template request", e); } @@ -350,7 +387,9 @@ public PutIndexTemplateRequest source(Map templateSource) { /** * The template source definition. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest source(String templateSource) { return source(XContentHelper.convertToMap(XContentFactory.xContent(templateSource), templateSource, true)); } @@ -358,6 +397,15 @@ public PutIndexTemplateRequest source(String templateSource) { /** * The template source definition. */ + public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source) { return source(source, 0, source.length); } @@ -365,6 +413,15 @@ public PutIndexTemplateRequest source(byte[] source) { /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -372,10 +429,26 @@ public PutIndexTemplateRequest source(byte[] source, int offset, int length) { /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(BytesReference source) { return source(XContentHelper.convertToMap(source, true).v2()); } + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2()); + } + public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { customs.put(custom.type(), custom); return this; @@ -471,7 +544,14 @@ public void readFrom(StreamInput in) throws IOException { settings = readSettingsFromStream(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + final String type = in.readString(); + String mappingSource = in.readString(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported + // we do not know the incoming type so convert it if needed + mappingSource = + XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource)); + } + mappings.put(type, mappingSource); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index c1db96ae7ce5c..e7ca25e51191b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Collections; import java.util.List; @@ -100,15 +101,25 @@ public PutIndexTemplateRequestBuilder setSettings(Settings.Builder settings) { } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSettings(String source) { request.settings(source); return this; } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) + */ + public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** + * The settings to crete the index template with (either json or yaml format) */ public PutIndexTemplateRequestBuilder setSettings(Map source) { request.settings(source); @@ -120,12 +131,26 @@ public PutIndexTemplateRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } + /** + * Adds mapping that will be added when the index template gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type/format of the source + */ + public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). @@ -226,7 +251,9 @@ public PutIndexTemplateRequestBuilder setSource(Map templateSource) { /** * The template source definition. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(String templateSource) { request.source(templateSource); return this; @@ -235,6 +262,16 @@ public PutIndexTemplateRequestBuilder setSource(String templateSource) { /** * The template source definition. */ + public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + + /** + * The template source definition. + * @deprecated use {@link #setSource(BytesReference, XContentType)} + */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) { request.source(templateSource); return this; @@ -242,7 +279,9 @@ public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) { /** * The template source definition. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) { request.source(templateSource); return this; @@ -251,8 +290,26 @@ public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) { /** * The template source definition. */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + + /** + * The template source definition. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length) { request.source(templateSource, offset, length); return this; } + + /** + * The template source definition. + */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length, XContentType xContentType) { + request.source(templateSource, offset, length, xContentType); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 20d5e64f49a7f..371659586f9af 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -244,7 +245,9 @@ public long estimatedSizeInBytes() { /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequest add(byte[] data, int from, int length) throws IOException { return add(data, from, length, null, null); } @@ -252,6 +255,15 @@ public BulkRequest add(byte[] data, int from, int length) throws IOException { /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException { + return add(data, from, length, null, null, xContentType); + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(new BytesArray(data, from, length), defaultIndex, defaultType); } @@ -259,6 +271,17 @@ public BulkRequest add(byte[] data, int from, int length, @Nullable String defau /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(new BytesArray(data, from, length), defaultIndex, defaultType, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, true); } @@ -266,12 +289,40 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null /** * Adds a framed data in binary format */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, true, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, boolean, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex); } + /** + * Adds a framed data in binary format + */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex, xContentType); + } + + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws IOException { - XContent xContent = XContentFactory.xContent(data); + XContentType xContentType = XContentFactory.xContentType(data); + return add(data, defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, payload, + allowExplicitIndex, xContentType); + } + + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String + defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String + defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex, XContentType xContentType) throws IOException { + XContent xContent = xContentType.xContent(); int line = 0; int from = 0; int length = data.length(); @@ -387,16 +438,16 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null if ("index".equals(action)) { if (opType == null) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) - .setPipeline(pipeline).source(data.slice(from, nextMarker - from)), payload); + .setPipeline(pipeline).source(data.slice(from, nextMarker - from), xContentType), payload); } else { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create("create".equals(opType)).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } } else if ("create".equals(action)) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create(true).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } else if ("update".equals(action)) { UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict) .version(version).versionType(versionType) diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index c48a8f507b862..8f634fa28a41b 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes @@ -97,7 +98,9 @@ public BulkRequestBuilder add(UpdateRequestBuilder request) { /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length) throws Exception { request.add(data, from, length, null, null); return this; @@ -106,11 +109,30 @@ public BulkRequestBuilder add(byte[] data, int from, int length) throws Exceptio /** * Adds a framed data in binary format */ + public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception { + request.add(data, from, length, null, null, xContentType); + return this; + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception { request.add(data, from, length, defaultIndex, defaultType); return this; } + /** + * Adds a framed data in binary format + */ + public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws Exception { + request.add(data, from, length, defaultIndex, defaultType, xContentType); + return this; + } + /** * Sets the number of shard copies that must be active before proceeding with the write. * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 12981664c2a66..f5faa1840365e 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -328,7 +328,7 @@ private UpdateResultHolder executeUpdateRequest(UpdateRequest updateRequest, Ind if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) || (updateRequest.fields() != null && updateRequest.fields().length > 0)) { Tuple> sourceAndContent = - XContentHelper.convertToMap(indexSourceAsBytes, true); + XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType()); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } @@ -427,8 +427,9 @@ private Translog.Location locationToSync(Translog.Location current, Translog.Loc */ public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) throws IOException { final ShardId shardId = replica.shardId(); - SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source()) - .routing(request.routing()).parent(request.parent()); + SourceToParse sourceToParse = + SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source(), + request.getContentType()).routing(request.routing()).parent(request.parent()); final Engine.Index operation; try { @@ -445,8 +446,9 @@ public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest reque /** Utility method to prepare an index operation on primary shards */ static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) { - SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source()) - .routing(request.routing()).parent(request.parent()); + SourceToParse sourceToParse = + SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(), + request.getContentType()).routing(request.routing()).parent(request.parent()); return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry()); } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java index 8831b2f63682d..4f2aefc1011c0 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java @@ -37,6 +37,8 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.search.SearchHit; @@ -232,6 +234,10 @@ public BytesReference getSource() { return source; } + @Override + public XContentType getXContentType() { + return XContentFactory.xContentType(source); + } @Override public long getVersion() { return delegate.getVersion(); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java index 8b8a350ffdc78..73aa653698611 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; @@ -80,7 +81,7 @@ public final void startNextScroll(TimeValue extraKeepAlive, Consumer o }); } protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer onResponse); - + @Override public final void close() { String scrollId = this.scrollId.get(); @@ -189,6 +190,10 @@ public interface Hit { * all. */ @Nullable BytesReference getSource(); + /** + * The content type of the hit source. Returns null if the source didn't come back from the search. + */ + @Nullable XContentType getXContentType(); /** * The document id of the parent of the hit if there is a parent or null if there isn't. */ @@ -209,6 +214,7 @@ public static class BasicHit implements Hit { private final long version; private BytesReference source; + private XContentType xContentType; private String parent; private String routing; @@ -244,8 +250,14 @@ public BytesReference getSource() { return source; } - public BasicHit setSource(BytesReference source) { + @Override + public XContentType getXContentType() { + return xContentType; + } + + public BasicHit setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; return this; } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index d1a9927c67c5e..7e3f14368debe 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -47,6 +47,7 @@ import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -55,10 +56,10 @@ * created using {@link org.elasticsearch.client.Requests#indexRequest(String)}. * * The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and - * {@link #source(byte[])} to be set. + * {@link #source(byte[], XContentType)} to be set. * - * The source (content to index) can be set in its bytes form using ({@link #source(byte[])}), - * its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} + * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}), + * its string form ({@link #source(String, XContentType)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} * ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}). * * If the {@link #id(String)} is not set, it will be automatically generated. @@ -83,7 +84,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; - private XContentType contentType = Requests.INDEX_CONTENT_TYPE; + private XContentType contentType; private String pipeline; @@ -103,7 +104,7 @@ public IndexRequest() { /** * Constructs a new index request against the specific index. The {@link #type(String)} - * {@link #source(byte[])} must be set. + * {@link #source(byte[], XContentType)} must be set. */ public IndexRequest(String index) { this.index = index; @@ -140,7 +141,9 @@ public ActionRequestValidationException validate() { if (source == null) { validationException = addValidationError("source is missing", validationException); } - + if (contentType == null) { + validationException = addValidationError("content type is missing", validationException); + } final long resolvedVersion = resolveVersionDefaults(); if (opType() == OpType.CREATE) { if (versionType != VersionType.INTERNAL) { @@ -179,20 +182,13 @@ public ActionRequestValidationException validate() { } /** - * The content type that will be used when generating a document from user provided objects like Maps. + * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the + * source at index time */ public XContentType getContentType() { return contentType; } - /** - * Sets the content type that will be used when generating a document from user provided objects (like Map). - */ - public IndexRequest contentType(XContentType contentType) { - this.contentType = contentType; - return this; - } - /** * The type of the indexed document. */ @@ -284,16 +280,16 @@ public BytesReference source() { } public Map sourceAsMap() { - return XContentHelper.convertToMap(source, false).v2(); + return XContentHelper.convertToMap(source, false, contentType).v2(); } /** - * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}. + * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format * * @param source The map to index */ public IndexRequest source(Map source) throws ElasticsearchGenerationException { - return source(source, contentType); + return source(source, Requests.INDEX_CONTENT_TYPE); } /** @@ -314,24 +310,32 @@ public IndexRequest source(Map source, XContentType contentType) throws Elastics /** * Sets the document source to index. * - * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #source(byte[])}. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public IndexRequest source(String source) { - this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); - return this; + return source(new BytesArray(source), XContentFactory.xContentType(source)); + } + + /** + * Sets the document source to index. + * + * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #source(byte[], XContentType)}. + */ + public IndexRequest source(String source, XContentType xContentType) { + return source(new BytesArray(source), xContentType); } /** * Sets the content source to index. */ public IndexRequest source(XContentBuilder sourceBuilder) { - source = sourceBuilder.bytes(); - return this; + return source(sourceBuilder.bytes(), sourceBuilder.contentType()); } /** - * Sets the content source to index. + * Sets the content source to index using the default content type ({@link Requests#INDEX_CONTENT_TYPE}) *

* Note: the number of objects passed to this method must be an even * number. Also the first argument in each pair (the field name) must have a @@ -339,6 +343,18 @@ public IndexRequest source(XContentBuilder sourceBuilder) { *

*/ public IndexRequest source(Object... source) { + return source(Requests.INDEX_CONTENT_TYPE, source); + } + + /** + * Sets the content source to index. + *

+ * Note: the number of objects passed to this method as varargs must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation. + *

+ */ + public IndexRequest source(XContentType xContentType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -346,7 +362,7 @@ public IndexRequest source(Object... source) { throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)"); } try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = XContentFactory.contentBuilder(xContentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -360,19 +376,39 @@ public IndexRequest source(Object... source) { /** * Sets the document to index in bytes form. + * @deprecated use {@link #source(BytesReference, XContentType)} */ + @Deprecated public IndexRequest source(BytesReference source) { - this.source = source; + return source(source, XContentFactory.xContentType(source)); + + } + + /** + * Sets the document to index in bytes form. + */ + public IndexRequest source(BytesReference source, XContentType xContentType) { + this.source = Objects.requireNonNull(source); + this.contentType = Objects.requireNonNull(xContentType); return this; } /** * Sets the document to index in bytes form. + * @deprecated use {@link #source(byte[], XContentType)} */ + @Deprecated public IndexRequest source(byte[] source) { return source(source, 0, source.length); } + /** + * Sets the document to index in bytes form. + */ + public IndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). @@ -380,10 +416,23 @@ public IndexRequest source(byte[] source) { * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #source(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequest source(byte[] source, int offset, int length) { - this.source = new BytesArray(source, offset, length); - return this; + return source(new BytesArray(source, offset, length), XContentFactory.xContentType(source)); + } + + /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + */ + public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); } /** @@ -515,6 +564,11 @@ public void readFrom(StreamInput in) throws IOException { pipeline = in.readOptionalString(); isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + contentType = in.readOptionalWriteable(XContentType::readFrom); + } else { + contentType = XContentFactory.xContentType(source); + } } @Override @@ -543,6 +597,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(pipeline); out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + out.writeOptionalWriteable(contentType); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index f7df8bffced3d..7af43ec35eccb 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -82,12 +82,22 @@ public IndexRequestBuilder setParent(String parent) { /** * Sets the source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; } + /** + * Sets the source. + */ + public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Index the Map as a JSON. * @@ -112,13 +122,26 @@ public IndexRequestBuilder setSource(Map source, XContentType content * Sets the document source to index. *

* Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #setSource(byte[])}. + * or using the {@link #setSource(byte[], XContentType)}. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(String source) { request.source(source); return this; } + /** + * Sets the document source to index. + *

+ * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #setSource(byte[], XContentType)}. + */ + public IndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Sets the content source to index. */ @@ -129,12 +152,22 @@ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) { /** * Sets the document to index in bytes form. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source) { request.source(source); return this; } + /** + * Sets the document to index in bytes form. + */ + public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). @@ -142,12 +175,28 @@ public IndexRequestBuilder setSource(byte[] source) { * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; } + /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + * @param xContentType The type/format of the source + */ + public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + /** * Constructs a simple document with a field name and value pairs. *

@@ -162,10 +211,15 @@ public IndexRequestBuilder setSource(Object... source) { } /** - * The content type that will be used to generate a document from user provided objects (like Map). + * Constructs a simple document with a field name and value pairs. + *

+ * Note: the number of objects passed as varargs to this method must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation. + *

*/ - public IndexRequestBuilder setContentType(XContentType contentType) { - request.contentType(contentType); + public IndexRequestBuilder setSource(XContentType xContentType, Object... source) { + request.source(xContentType, source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 10416146ba853..a28950b24c189 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -19,32 +19,40 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.action.ValidateActions.addValidationError; - public class PutPipelineRequest extends AcknowledgedRequest { private String id; private BytesReference source; + private XContentType xContentType; + /** + * Create a new pipeline request + * @deprecated use {@link #PutPipelineRequest(String, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public PutPipelineRequest(String id, BytesReference source) { - if (id == null) { - throw new IllegalArgumentException("id is missing"); - } - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } + this(id, source, XContentFactory.xContentType(source)); + } - this.id = id; - this.source = source; + /** + * Create a new pipeline request with the id and source along with the content type of the source + */ + public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { + this.id = Objects.requireNonNull(id); + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } PutPipelineRequest() { @@ -63,11 +71,20 @@ public BytesReference getSource() { return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); source = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -75,5 +92,8 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java index bd927115fb5ff..c03b3b84f8b5b 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class PutPipelineRequestBuilder extends ActionRequestBuilder { @@ -29,8 +30,13 @@ public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction a super(client, action, new PutPipelineRequest()); } + @Deprecated public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source) { super(client, action, new PutPipelineRequest(id, source)); } + public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source, + XContentType xContentType) { + super(client, action, new PutPipelineRequest(id, source, xContentType)); + } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index ef7b5e3d5bbed..362d2e7ab98ca 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Pipeline; @@ -34,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.ingest.IngestDocument.MetaData; @@ -42,12 +46,23 @@ public class SimulatePipelineRequest extends ActionRequest { private String id; private boolean verbose; private BytesReference source; + private XContentType xContentType; + /** + * Create a new request + * @deprecated use {@link #SimulatePipelineRequest(BytesReference, XContentType)} that does not attempt content autodetection + */ + @Deprecated public SimulatePipelineRequest(BytesReference source) { - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - this.source = source; + this(source, XContentFactory.xContentType(source)); + } + + /** + * Creates a new request with the given source and its content type + */ + public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } SimulatePipelineRequest() { @@ -78,12 +93,21 @@ public BytesReference getSource() { return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -92,6 +116,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } public static final class Fields { diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java index 4a13fa111e6a2..bb5d0e4e40003 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -22,22 +22,46 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class SimulatePipelineRequestBuilder extends ActionRequestBuilder { + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action) { super(client, action, new SimulatePipelineRequest()); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + * @deprecated use {@link #SimulatePipelineRequestBuilder(ElasticsearchClient, SimulatePipelineAction, BytesReference, XContentType)} to + * avoid content type auto-detection on the source bytes + */ + @Deprecated public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) { super(client, action, new SimulatePipelineRequest(source)); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ + public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source, + XContentType xContentType) { + super(client, action, new SimulatePipelineRequest(source, xContentType)); + } + + /** + * Set the id for the pipeline to simulate + */ public SimulatePipelineRequestBuilder setId(String id) { request.setId(id); return this; } + /** + * Enable or disable verbose mode + */ public SimulatePipelineRequestBuilder setVerbose(boolean verbose) { request.setVerbose(verbose); return this; diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java index 61fd400a1d372..3f67007df690d 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java @@ -47,7 +47,7 @@ public SimulatePipelineTransportAction(Settings settings, ThreadPool threadPool, @Override protected void doExecute(SimulatePipelineRequest request, ActionListener listener) { - final Map source = XContentHelper.convertToMap(request.getSource(), false).v2(); + final Map source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); final SimulatePipelineRequest.Parsed simulateRequest; try { diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index b83713e3a6afc..544b14523bc70 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.termvectors; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.RealtimeRequest; import org.elasticsearch.action.ValidateActions; @@ -33,7 +34,9 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import java.io.IOException; @@ -63,6 +66,8 @@ public class TermVectorsRequest extends SingleShardRequest i private BytesReference doc; + private XContentType xContentType; + private String routing; private String parent; @@ -156,8 +161,9 @@ public TermVectorsRequest(TermVectorsRequest other) { super(other.index()); this.id = other.id(); this.type = other.type(); - if (this.doc != null) { + if (other.doc != null) { this.doc = new BytesArray(other.doc().toBytesRef(), true); + this.xContentType = other.xContentType; } this.flagsEnum = other.getFlags().clone(); this.preference = other.preference(); @@ -225,22 +231,36 @@ public BytesReference doc() { return doc; } + public XContentType xContentType() { + return xContentType; + } + /** * Sets an artificial document from which term vectors are requested for. */ public TermVectorsRequest doc(XContentBuilder documentBuilder) { - return this.doc(documentBuilder.bytes(), true); + return this.doc(documentBuilder.bytes(), true, documentBuilder.contentType()); } /** * Sets an artificial document from which term vectors are requested for. + * @deprecated use {@link #doc(BytesReference, boolean, XContentType)} to avoid content auto detection */ + @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { + return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc)); + } + + /** + * Sets an artificial document from which term vectors are requested for. + */ + public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, XContentType xContentType) { // assign a random id to this artificial document, for routing if (generateRandomId) { this.id(String.valueOf(randomInt.getAndAdd(1))); } this.doc = doc; + this.xContentType = xContentType; return this; } @@ -479,6 +499,11 @@ public void readFrom(StreamInput in) throws IOException { if (in.readBoolean()) { doc = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } routing = in.readOptionalString(); parent = in.readOptionalString(); @@ -519,6 +544,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } out.writeOptionalString(routing); out.writeOptionalString(parent); diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 45e4aa8afdd42..4e2a7b466bc7c 100644 --- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -186,7 +186,8 @@ public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult()); if ((request.fetchSource() != null && request.fetchSource().fetchSource()) || (request.fields() != null && request.fields().length > 0)) { - Tuple> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true); + Tuple> sourceAndContent = + XContentHelper.convertToMap(upsertSourceBytes, true, upsertRequest.getContentType()); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes)); } else { update.setGetResult(null); diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 0d2801e44a003..504a297627d63 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -29,8 +29,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -52,8 +50,6 @@ public class UpdateRequest extends InstanceShardOperationRequest implements DocWriteRequest, WriteRequest { - private static final DeprecationLogger DEPRECATION_LOGGER = - new DeprecationLogger(Loggers.getLogger(UpdateRequest.class)); private String type; private String id; @@ -559,7 +555,9 @@ public UpdateRequest doc(Map source, XContentType contentType) { /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(String, XContentType)} */ + @Deprecated public UpdateRequest doc(String source) { safeDoc().source(source); return this; @@ -568,6 +566,16 @@ public UpdateRequest doc(String source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(String source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source) { safeDoc().source(source); return this; @@ -576,11 +584,29 @@ public UpdateRequest doc(byte[] source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(byte[] source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source, int offset, int length) { safeDoc().source(source, offset, length); return this; } + /** + * Sets the doc to use for updates when a script is not specified. + */ + public UpdateRequest doc(byte[] source, int offset, int length, XContentType xContentType) { + safeDoc().source(source, offset, length, xContentType); + return this; + } + /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. @@ -591,10 +617,11 @@ public UpdateRequest doc(Object... source) { } /** - * Sets the doc to use for updates when a script is not specified. + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. */ - public UpdateRequest doc(String field, Object value) { - safeDoc().source(field, value); + public UpdateRequest doc(XContentType xContentType, Object... source) { + safeDoc().source(xContentType, source); return this; } @@ -644,7 +671,9 @@ public UpdateRequest upsert(Map source, XContentType contentType) { /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(String, XContentType)} */ + @Deprecated public UpdateRequest upsert(String source) { safeUpsertRequest().source(source); return this; @@ -653,6 +682,16 @@ public UpdateRequest upsert(String source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(String source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source) { safeUpsertRequest().source(source); return this; @@ -661,11 +700,29 @@ public UpdateRequest upsert(byte[] source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(byte[] source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source, int offset, int length) { safeUpsertRequest().source(source, offset, length); return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequest upsert(byte[] source, int offset, int length, XContentType xContentType) { + safeUpsertRequest().source(source, offset, length, xContentType); + return this; + } + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. @@ -675,6 +732,15 @@ public UpdateRequest upsert(Object... source) { return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequest upsert(XContentType xContentType, Object... source) { + safeUpsertRequest().source(xContentType, source); + return this; + } + public IndexRequest upsertRequest() { return this.upsertRequest; } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 50d84a241291f..1a4d4077b1045 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -223,7 +223,9 @@ public UpdateRequestBuilder setDoc(Map source, XContentType contentType) { /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setDoc(String source) { request.doc(source); return this; @@ -232,6 +234,16 @@ public UpdateRequestBuilder setDoc(String source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(String source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source) { request.doc(source); return this; @@ -240,6 +252,16 @@ public UpdateRequestBuilder setDoc(byte[] source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) { request.doc(source, offset, length); return this; @@ -248,8 +270,8 @@ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) { /** * Sets the doc to use for updates when a script is not specified. */ - public UpdateRequestBuilder setDoc(String field, Object value) { - request.doc(field, value); + public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) { + request.doc(source, offset, length, xContentType); return this; } @@ -262,6 +284,15 @@ public UpdateRequestBuilder setDoc(Object... source) { return this; } + /** + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. + */ + public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) { + request.doc(xContentType, source); + return this; + } + /** * Sets the index request to be used if the document does not exists. Otherwise, a {@link org.elasticsearch.index.engine.DocumentMissingException} * is thrown. @@ -297,7 +328,9 @@ public UpdateRequestBuilder setUpsert(Map source, XContentType contentType) { /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setUpsert(String source) { request.upsert(source); return this; @@ -306,6 +339,16 @@ public UpdateRequestBuilder setUpsert(String source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source) { request.upsert(source); return this; @@ -314,11 +357,29 @@ public UpdateRequestBuilder setUpsert(byte[] source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) { request.upsert(source, offset, length); return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) { + request.upsert(source, offset, length, xContentType); + return this; + } + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. @@ -328,6 +389,15 @@ public UpdateRequestBuilder setUpsert(Object... source) { return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) { + request.upsert(xContentType, source); + return this; + } + /** * Sets whether the specified doc parameter should be used as upsert document. */ diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 14abc77513a13..3f705a215ea11 100644 --- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -112,6 +112,7 @@ import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; /** @@ -545,9 +546,16 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Stores an ingest pipeline + * @deprecated use {@link #preparePutPipeline(String, BytesReference, XContentType)} */ + @Deprecated PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source); + /** + * Stores an ingest pipeline + */ + PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType); + /** * Deletes a stored ingest pipeline */ @@ -596,8 +604,14 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Simulates an ingest pipeline */ + @Deprecated SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source); + /** + * Simulates an ingest pipeline + */ + SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType); + /** * Explain the allocation of a shard */ diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index b93c00ca9fef2..5a789af3eebae 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -343,6 +343,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -1081,6 +1082,11 @@ public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference so return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source); } + @Override + public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType) { + return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source, xContentType); + } + @Override public void deletePipeline(DeletePipelineRequest request, ActionListener listener) { execute(DeletePipelineAction.INSTANCE, request, listener); @@ -1131,6 +1137,11 @@ public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference sou return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source); } + @Override + public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) { + return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source, xContentType); + } + @Override public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener listener) { execute(ClusterAllocationExplainAction.INSTANCE, request, listener); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 4ba244e107da8..5bba34904d0c8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 0efb640c5a768..0bde4a23b032f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -265,11 +265,13 @@ public ClusterState execute(ClusterState currentState) throws Exception { for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); for (ObjectObjectCursor cursor : template.mappings()) { + String mappingString = cursor.value.string(); if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), - MapperService.parseMapping(xContentRegistry, cursor.value.string())); + MapperService.parseMapping(xContentRegistry, mappingString)); } else { - mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, cursor.value.string())); + mappings.put(cursor.key, + MapperService.parseMapping(xContentRegistry, mappingString)); } } // handle custom diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 279be58e01d4d..69adc5878ff01 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -237,6 +237,7 @@ public void apply(Settings value, Settings current, Settings previous) { HttpTransportSettings.SETTING_CORS_ALLOW_METHODS, HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS, HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED, + HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH, HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE, HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE, diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index 883f1e7c82c1c..579be7ce31fa1 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.InputStream; @@ -960,7 +961,9 @@ public Builder put(Dictionary properties) { /** * Loads settings from the actual string content that represents them using the * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * @deprecated use {@link #loadFromSource(String, XContentType)} to avoid content type detection */ + @Deprecated public Builder loadFromSource(String source) { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromSource(source); try { @@ -972,9 +975,24 @@ public Builder loadFromSource(String source) { return this; } + /** + * Loads settings from the actual string content that represents them using the + * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader + */ + public Builder loadFromSource(String source, XContentType xContentType) { + SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType); + try { + Map loadedSettings = settingsLoader.load(source); + put(loadedSettings); + } catch (Exception e) { + throw new SettingsException("Failed to load settings from [" + source + "]", e); + } + return this; + } + /** * Loads settings from a url that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromPath(Path path) throws IOException { // NOTE: loadFromStream will close the input stream @@ -983,7 +1001,7 @@ public Builder loadFromPath(Path path) throws IOException { /** * Loads settings from a stream that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromStream(String resourceName, InputStream is) throws IOException { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName); diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java index 5f2da22c5f2dd..5d8cb4918b2ea 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java +++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.settings.loader; +import org.elasticsearch.common.xcontent.XContentType; + /** * A class holding factory methods for settings loaders that attempts * to infer the type of the underlying settings content. @@ -33,9 +35,7 @@ private SettingsLoaderFactory() { * name. This factory method assumes that if the resource name ends * with ".json" then the content should be parsed as JSON, else if * the resource name ends with ".yml" or ".yaml" then the content - * should be parsed as YAML, else if the resource name ends with - * ".properties" then the content should be parsed as properties, - * otherwise default to attempting to parse as JSON. Note that the + * should be parsed as YAML, otherwise throws an exception. Note that the * parsers returned by this method will not accept null-valued * keys. * @@ -59,13 +59,15 @@ public static SettingsLoader loaderFromResource(String resourceName) { * contains an opening and closing brace ('{' and '}') then the * content should be parsed as JSON, else if the underlying content * fails this condition but contains a ':' then the content should - * be parsed as YAML, and otherwise should be parsed as properties. + * be parsed as YAML, and otherwise throws an exception. * Note that the JSON and YAML parsers returned by this method will * accept null-valued keys. * * @param source The underlying settings content. * @return A settings loader. + * @deprecated use {@link #loaderFromXContentType(XContentType)} instead */ + @Deprecated public static SettingsLoader loaderFromSource(String source) { if (source.indexOf('{') != -1 && source.indexOf('}') != -1) { return new JsonSettingsLoader(true); @@ -76,4 +78,20 @@ public static SettingsLoader loaderFromSource(String source) { } } + /** + * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only {@link XContentType#JSON} and + * {@link XContentType#YAML} are supported + * + * @param xContentType The content type + * @return A settings loader. + */ + public static SettingsLoader loaderFromXContentType(XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return new JsonSettingsLoader(true); + } else if (xContentType == XContentType.YAML) { + return new YamlSettingsLoader(true); + } else { + throw new IllegalArgumentException("unsupported content type [" + xContentType + "]"); + } + } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index df34ec726fd86..189e9d3c8d5dc 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -964,21 +964,60 @@ public XContentBuilder byteSizeField(String rawFieldName, String readableFieldNa // Raw fields ////////////////////////////////// + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #rawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawField(String name, InputStream value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the value taken from the bytes in the stream + */ + public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #rawField(String name, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawField(String name, BytesReference value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the given bytes as the value + */ + public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #rawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawValue(BytesReference value) throws IOException { generator.writeRawValue(value); return this; } + /** + * Writes a value with the source coming directly from the bytes + */ + public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException { + generator.writeRawValue(value, contentType); + return this; + } + public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException { generator.copyCurrentStructure(parser); return this; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 478f3a8a08f82..60a188ca6ce58 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -86,12 +86,42 @@ public interface XContentGenerator extends Closeable, Flushable { void writeBinary(byte[] value, int offset, int length) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #writeRawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, InputStream value) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + */ + void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException; + + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, BytesReference value) throws IOException; + /** + * Writes a raw field with the given bytes as the value + */ + void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException; + + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawValue(BytesReference value) throws IOException; + /** + * Writes a value with the source coming directly from the bytes + */ + void writeRawValue(BytesReference value, XContentType xContentType) throws IOException; + void copyCurrentStructure(XContentParser parser) throws IOException; /** diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 6cf8d12c21c8d..dd7508280d9ca 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -41,6 +41,11 @@ @SuppressWarnings("unchecked") public class XContentHelper { + /** + * Creates a parser based on the bytes provided + * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException { Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -48,17 +53,49 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - XContentType contentType = XContentFactory.xContentType(compressedInput); + final XContentType contentType = XContentFactory.xContentType(compressedInput); return XContentFactory.xContent(contentType).createParser(xContentRegistry, compressedInput); } else { return XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes.streamInput()); } } + /** + * Creates a parser for the bytes using the supplied content-type + */ + public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes, + XContentType xContentType) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + InputStream compressedInput = compressor.streamInput(bytes.streamInput()); + if (compressedInput.markSupported() == false) { + compressedInput = new BufferedInputStream(compressedInput); + } + return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput); + } else { + return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput()); + } + } + + /** + * Converts the given bytes into a map that is optionally ordered. + * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)} + * instead with the proper {@link XContentType} + */ + @Deprecated public static Tuple> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { + return convertToMap(bytes, ordered, null); + } + + /** + * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null. + */ + public static Tuple> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) + throws ElasticsearchParseException { try { - XContentType contentType; + final XContentType contentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -66,13 +103,12 @@ public static Tuple> convertToMap(BytesReferen if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); } - contentType = XContentFactory.xContentType(compressedStreamInput); input = compressedStreamInput; } else { - contentType = XContentFactory.xContentType(bytes); input = bytes.streamInput(); } - return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); + contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), input, ordered)); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); } @@ -105,15 +141,27 @@ public static Map convertToMap(XContent xContent, InputStream in } } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException { return convertToJson(bytes, reformatJson, false); } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - XContentType xContentType = XContentFactory.xContentType(bytes); + return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes)); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException { + return convertToJson(bytes, reformatJson, false, xContentType); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType) + throws IOException { + Objects.requireNonNull(xContentType); if (xContentType == XContentType.JSON && !reformatJson) { return bytes.utf8ToString(); } + // It is safe to use EMPTY here because this never uses namedObject try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY, bytes.streamInput())) { @@ -365,7 +413,10 @@ public static void copyCurrentEvent(XContentGenerator generator, XContentParser /** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}. + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type + * auto-detection */ + @Deprecated public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { @@ -376,6 +427,22 @@ public static void writeRawField(String field, BytesReference source, XContentBu } } + /** + * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using + * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference, XContentType)}. + */ + public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder, + ToXContent.Params params) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(source); + if (compressor != null) { + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + builder.rawField(field, compressedStreamInput, xContentType); + } else { + builder.rawField(field, source, xContentType); + } + } + /** * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided * {@link XContentType}. Wraps the output into a new anonymous object. diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index ddd736e0d0070..8e3c298270444 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Locale; +import java.util.Objects; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -40,7 +41,7 @@ public enum XContentType implements Writeable { */ JSON(0) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/json"; } @@ -64,7 +65,7 @@ public XContent xContent() { */ SMILE(1) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/smile"; } @@ -83,7 +84,7 @@ public XContent xContent() { */ YAML(2) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/yaml"; } @@ -102,7 +103,7 @@ public XContent xContent() { */ CBOR(3) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/cbor"; } @@ -117,12 +118,18 @@ public XContent xContent() { } }; + /** + * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has + * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method + * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a + * format query string parameter. This method will return {@code null} if no match is found + */ public static XContentType fromMediaTypeOrFormat(String mediaType) { if (mediaType == null) { return null; } for (XContentType type : values()) { - if (isSameMediaTypeAs(mediaType, type)) { + if (isSameMediaTypeOrFormatAs(mediaType, type)) { return type; } } @@ -133,7 +140,22 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) { return null; } - private static boolean isSameMediaTypeAs(String stringType, XContentType type) { + /** + * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. + * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type} + * HTTP header. This method will return {@code null} if no match is found + */ + public static XContentType fromMediaType(String mediaType) { + final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT); + for (XContentType type : values()) { + if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) { + return type; + } + } + return null; + } + + private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) { return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) || stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") || type.shortName().equalsIgnoreCase(stringType); @@ -157,7 +179,7 @@ public String mediaType() { public abstract XContent xContent(); - protected abstract String mediaTypeWithoutParameters(); + public abstract String mediaTypeWithoutParameters(); public static XContentType readFrom(StreamInput in) throws IOException { int index = in.readVInt(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 07ae16b96c69b..1e09f8334f772 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -307,6 +307,11 @@ public void writeRawField(String name, InputStream content) throws IOException { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public void writeRawField(String name, InputStream content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { // EMPTY is safe here because we never call namedObject when writing raw data try (XContentParser parser = XContentFactory.xContent(contentType).createParser(NamedXContentRegistry.EMPTY, content)) { @@ -328,6 +333,11 @@ public final void writeRawField(String name, BytesReference content) throws IOEx if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public final void writeRawField(String name, BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { writeFieldName(name); copyRawValue(content, contentType.xContent()); @@ -345,6 +355,11 @@ public final void writeRawValue(BytesReference content) throws IOException { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawValue(content, contentType); + } + + @Override + public final void writeRawValue(BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { copyRawValue(content, contentType.xContent()); } else { diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 60bc3449d0be5..b5e254aa4c2ef 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -68,6 +68,8 @@ public final class HttpTransportSettings { Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope); public static final Setting SETTING_HTTP_DETAILED_ERRORS_ENABLED = Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope); + public static final Setting SETTING_HTTP_CONTENT_TYPE_REQUIRED = + Setting.boolSetting("http.content_type.required", false, Property.NodeScope); public static final Setting SETTING_HTTP_MAX_CONTENT_LENGTH = Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope); public static final Setting SETTING_HTTP_MAX_CHUNK_SIZE = diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 8672bbc8351ea..b1d8ac188626f 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -190,7 +190,7 @@ public String toString() { return sb.toString(); } try { - String source = XContentHelper.convertToJson(doc.source(), reformat); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]"); } catch (IOException e) { sb.append(", source[_failed_to_convert_]"); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index fdc45530b9976..a8c74101bff45 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -256,8 +257,9 @@ public Map objectMappers() { return this.objectMappers; } + // TODO this method looks like it is only used in tests... public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException { - return parse(SourceToParse.source(index, type, id, source)); + return parse(SourceToParse.source(index, type, id, source, XContentType.JSON)); } public ParsedDocument parse(SourceToParse source) throws MapperParsingException { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 031f139075643..be25775c1353e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -24,9 +24,9 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.QueryShardContext; @@ -80,7 +80,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { - Map root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); + Map root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2(); Tuple> t = extractMapping(type, root); type = t.v1(); mapping = t.v2(); @@ -162,7 +162,7 @@ private static String getRemainingFields(Map map) { private Tuple> extractMapping(String type, String source) throws MapperParsingException { Map root; - try (XContentParser parser = XContentFactory.xContent(source).createParser(xContentRegistry, source)) { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, source)) { root = parser.mapOrdered(); } catch (Exception e) { throw new MapperParsingException("failed to parse mapping definition", e); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 014ff45200500..97891b28aa742 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -154,6 +154,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext. source.routing(), context.docs(), context.sourceToParse().source(), + context.sourceToParse().getXContentType(), update ).parent(source.parent()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index ee4b5fea15e0d..41b29f3c67f4b 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; @@ -37,6 +38,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -189,8 +191,11 @@ public DocumentMapperParser documentMapperParser() { return this.documentParser; } + /** + * Parses the mappings (formatted as JSON) into a map + */ public static Map parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception { - try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(xContentRegistry, mappingSource)) { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, mappingSource)) { return parser.map(); } } @@ -318,7 +323,8 @@ private synchronized Map internalMerge(Map documents; private BytesReference source; + private XContentType xContentType; private Mapping dynamicMappingsUpdate; @@ -55,6 +56,7 @@ public ParsedDocument(Field version, String routing, List documents, BytesReference source, + XContentType xContentType, Mapping dynamicMappingsUpdate) { this.version = version; this.seqID = seqID; @@ -65,6 +67,7 @@ public ParsedDocument(Field version, this.documents = documents; this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; + this.xContentType = xContentType; } public BytesRef uid() { @@ -105,8 +108,13 @@ public BytesReference source() { return this.source; } - public void setSource(BytesReference source) { + public XContentType getXContentType() { + return this.xContentType; + } + + public void setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; } public ParsedDocument parent(String parent) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index e06ec80a472dd..2eb2fdeaa0333 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -241,7 +241,8 @@ protected void parseCreateField(ParseContext context, List field if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(source, true); + Tuple> mapTuple = + XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType()); Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); XContentType contentType = mapTuple.v1(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 0cafc50bbe2ab..a8a983ecde832 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -23,15 +23,18 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; public class SourceToParse { - public static SourceToParse source(String index, String type, String id, BytesReference source) { - return source(Origin.PRIMARY, index, type, id, source); + public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) { + return source(Origin.PRIMARY, index, type, id, source, contentType); } - public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) { - return new SourceToParse(origin, index, type, id, source); + public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source, + XContentType contentType) { + return new SourceToParse(origin, index, type, id, source, contentType); } private final Origin origin; @@ -48,14 +51,17 @@ public static SourceToParse source(Origin origin, String index, String type, Str private String parentId; - private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) { + private XContentType xContentType; + + private SourceToParse(Origin origin, String index, String type, String id, BytesReference source, XContentType xContentType) { this.origin = Objects.requireNonNull(origin); this.index = Objects.requireNonNull(index); this.type = Objects.requireNonNull(type); this.id = Objects.requireNonNull(id); // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays - this.source = new BytesArray(source.toBytesRef()); + this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); + this.xContentType = Objects.requireNonNull(xContentType); } public Origin origin() { @@ -91,6 +97,10 @@ public String routing() { return this.routing; } + public XContentType getXContentType() { + return this.xContentType; + } + public SourceToParse routing(String routing) { this.routing = routing; return this; diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index bfb579e0c322c..ee0eff2a45162 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; @@ -161,6 +162,7 @@ public interface Field { private String type; private String id; private BytesReference doc; + private XContentType xContentType; private String[] fields; private Map perFieldAnalyzer; private String routing; @@ -178,6 +180,7 @@ public Item() { this.type = copy.type; this.id = copy.id; this.doc = copy.doc; + this.xContentType = copy.xContentType; this.fields = copy.fields; this.perFieldAnalyzer = copy.perFieldAnalyzer; this.version = copy.version; @@ -214,6 +217,7 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) this.index = index; this.type = type; this.doc = doc.bytes(); + this.xContentType = doc.contentType(); } /** @@ -224,6 +228,11 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) type = in.readOptionalString(); if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } else { id = in.readString(); } @@ -241,6 +250,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } else { out.writeString(id); } @@ -325,6 +337,10 @@ public Item versionType(VersionType versionType) { return this; } + XContentType xContentType() { + return xContentType; + } + /** * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document. */ @@ -342,7 +358,7 @@ public TermVectorsRequest toTermVectorsRequest() { .termStatistics(false); // for artificial docs to make sure that the id has changed in the item too if (doc != null) { - termVectorsRequest.doc(doc, true); + termVectorsRequest.doc(doc, true, xContentType); this.id = termVectorsRequest.id(); } return termVectorsRequest; @@ -366,6 +382,7 @@ public static Item parse(XContentParser parser, Item item) throws IOException { item.id = parser.text(); } else if (Field.DOC.match(currentFieldName)) { item.doc = jsonBuilder().copyCurrentStructure(parser).bytes(); + item.xContentType = XContentType.JSON; } else if (Field.FIELDS.match(currentFieldName)) { if (token == XContentParser.Token.START_ARRAY) { List fields = new ArrayList<>(); @@ -416,12 +433,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Field.ID.getPreferredName(), this.id); } if (this.doc != null) { - XContentType contentType = XContentFactory.xContentType(this.doc); - if (contentType == builder.contentType()) { - builder.rawField(Field.DOC.getPreferredName(), this.doc); - } else { - builder.rawField(Field.DOC.getPreferredName(), doc); - } + builder.rawField(Field.DOC.getPreferredName(), this.doc, xContentType); } if (this.fields != null) { builder.array(Field.FIELDS.getPreferredName(), this.fields); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index 6f4ae66a12a03..ad00e39097efe 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; @@ -157,7 +158,8 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio Translog.Index index = (Translog.Index) operation; // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all // autoGeneratedID docs that are coming from the primary are updated correctly. - Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source()) + Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), + source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 520cb13390f79..7981c89124280 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetField; @@ -270,7 +271,8 @@ private static Fields generateTermVectors(IndexShard indexShard, Map { XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); - builder.setConfig(contentBuilder.bytes()); + builder.setConfig(contentBuilder.bytes(), contentBuilder.contentType()); }, new ParseField("config"), ObjectParser.ValueType.OBJECT); + } public static ContextParser getParser() { @@ -56,17 +61,19 @@ private static class Builder { private String id; private BytesReference config; + private XContentType xContentType; void setId(String id) { this.id = id; } - void setConfig(BytesReference config) { + void setConfig(BytesReference config, XContentType xContentType) { this.config = config; + this.xContentType = xContentType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config); + return new PipelineConfiguration(id, config, xContentType); } } @@ -75,10 +82,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; + private final XContentType xContentType; - public PipelineConfiguration(String id, BytesReference config) { - this.id = id; - this.config = config; + public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + this.id = Objects.requireNonNull(id); + this.config = Objects.requireNonNull(config); + this.xContentType = Objects.requireNonNull(xContentType); } public String getId() { @@ -86,7 +95,17 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true).v2(); + return XContentHelper.convertToMap(config, true, xContentType).v2(); + } + + // pkg-private for tests + XContentType getXContentType() { + return xContentType; + } + + // pkg-private for tests + BytesReference getConfig() { + return config; } @Override @@ -99,7 +118,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference()); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in)); + } else { + final String id = in.readString(); + final BytesReference config = in.readBytesReference(); + return new PipelineConfiguration(id, config, XContentFactory.xContentType(config)); + } } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -110,6 +135,9 @@ public static Diff readDiffFrom(StreamInput in) throws IO public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java index 1171865a007d0..d476d7c2bd857 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java @@ -162,7 +162,7 @@ void validatePipeline(Map ingestInfos, PutPipelineReq throw new IllegalStateException("Ingest info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { @@ -185,7 +185,7 @@ ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metaData(MetaData.builder(currentState.getMetaData()) .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)) diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 4b3505e97e35c..bdc78c82dd503 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -58,28 +57,37 @@ protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.hasContent() ? request.content() : null, true); + return newBuilder(request.getXContentType(), true); } @Override public XContentBuilder newErrorBuilder() throws IOException { // Disable filtering when building error responses - return newBuilder(request.hasContent() ? request.content() : null, false); + return newBuilder(request.getXContentType(), false); } + /** + * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following + * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format + * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map + * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls + * back to JSON. + */ @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - XContentType contentType = XContentType.fromMediaTypeOrFormat(format); - if (contentType == null) { - // try and guess it from the auto detect source - if (autoDetectSource != null) { - contentType = XContentFactory.xContentType(autoDetectSource); + public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + // try to determine the response content type from the media type or the format query string parameter, with the format parameter + // taking precedence over the Accept header + XContentType responseContentType = XContentType.fromMediaTypeOrFormat(format); + if (responseContentType == null) { + if (requestContentType != null) { + // if there was a parsed content-type for the incoming request use that since no format was specified using the query + // string parameter or the HTTP Accept header + responseContentType = requestContentType; + } else { + // default to JSON output when all else fails + responseContentType = XContentType.JSON; } } - if (contentType == null) { - // default to JSON - contentType = XContentType.JSON; - } Set includes = Collections.emptySet(); Set excludes = Collections.emptySet(); @@ -89,7 +97,7 @@ public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boo excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet()); } - XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes); + XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), bytesOutput(), includes, excludes); if (pretty) { builder.prettyPrint().lfAtEnd(); } @@ -125,5 +133,4 @@ public RestRequest request() { public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } - } diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java index 2d45bc1765b98..a7b709625ce59 100644 --- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java +++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -146,6 +147,13 @@ private static XContentBuilder build(RestChannel channel, RestStatus status, Exc return builder; } + static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException { + return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject() + .field("error", errorMessage) + .field("status", status.getStatus()) + .endObject()); + } + public static ElasticsearchStatusException errorFromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index 2a56313fd8a79..8c8346f0ef4b2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -20,9 +20,9 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -35,7 +35,7 @@ public interface RestChannel { XContentBuilder newErrorBuilder() throws IOException; - XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException; + XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException; BytesStreamOutput bytesOutput(); @@ -47,5 +47,4 @@ public interface RestChannel { boolean detailedErrorsEnabled(); void sendResponse(RestResponse response); - } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 5ac82b7e4542e..fd3587df2aec2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -22,18 +22,19 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -42,12 +43,15 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.common.xcontent.XContentType; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.FORBIDDEN; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE; import static org.elasticsearch.rest.RestStatus.OK; public class RestController extends AbstractComponent { @@ -67,6 +71,10 @@ public class RestController extends AbstractComponent { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set headersToCopy; + private final boolean isContentTypeRequired; + + private final DeprecationLogger deprecationLogger; + public RestController(Settings settings, Set headersToCopy, UnaryOperator handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService) { super(settings); @@ -77,10 +85,10 @@ public RestController(Settings settings, Set headersToCopy, UnaryOperato this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; + this.isContentTypeRequired = HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.get(settings); + this.deprecationLogger = new DeprecationLogger(logger); } - - /** * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. * @@ -165,15 +173,22 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont } RestChannel responseChannel = channel; try { - int contentLength = request.content().length(); - if (canTripCircuitBreaker(request)) { - inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); + final int contentLength = request.hasContent() ? request.content().length() : 0; + assert contentLength >= 0 : "content length was negative, how is that possible?"; + final RestHandler handler = getHandler(request); + + if (contentLength > 0 && hasContentTypeOrCanAutoDetect(request, handler) == false) { + sendContentTypeErrorMessage(request, responseChannel); } else { - inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + if (canTripCircuitBreaker(request)) { + inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); + } else { + inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + } + // iff we could reserve bytes for the request we need to send the response also over this channel + responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); + dispatchRequest(request, responseChannel, client, threadContext, handler); } - // iff we could reserve bytes for the request we need to send the response also over this channel - responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); - dispatchRequest(request, responseChannel, client, threadContext); } catch (Exception e) { try { responseChannel.sendResponse(new BytesRestResponse(channel, e)); @@ -185,33 +200,75 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont } } - void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception { - if (!checkRequestParameters(request, channel)) { - return; - } - try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { - for (String key : headersToCopy) { - String httpHeader = request.header(key); - if (httpHeader != null) { - threadContext.putHeader(key, httpHeader); + void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext, + final RestHandler handler) throws Exception { + if (checkRequestParameters(request, channel) == false) { + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled.")); + } else { + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { + for (String key : headersToCopy) { + String httpHeader = request.header(key); + if (httpHeader != null) { + threadContext.putHeader(key, httpHeader); + } } - } - final RestHandler handler = getHandler(request); + if (handler == null) { + if (request.method() == RestRequest.Method.OPTIONS) { + // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - if (handler == null) { - if (request.method() == RestRequest.Method.OPTIONS) { - // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + } else { + final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; + channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + } } else { - final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; - channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); + wrappedHandler.handleRequest(request, channel, client); } + } + } + } + + /** + * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present, matches an + * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method + * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text + */ + private boolean hasContentTypeOrCanAutoDetect(final RestRequest restRequest, final RestHandler restHandler) { + if (restRequest.getXContentType() == null) { + if (restHandler != null && restHandler.supportsPlainText()) { + // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can + // be removed! + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else if (isContentTypeRequired) { + return false; } else { - final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); - wrappedHandler.handleRequest(request, channel, client); + deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using " + + "the [Content-Type] header."); + XContentType xContentType = XContentFactory.xContentType(restRequest.content()); + if (xContentType == null) { + return false; + } else { + restRequest.setXContentType(xContentType); + } } } + return true; + } + + private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) throws IOException { + final List contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); + final String errorMessage; + if (contentTypeHeader == null) { + errorMessage = "Content-Type header is missing"; + } else { + errorMessage = "Content-Type header [" + + Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; + } + + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(NOT_ACCEPTABLE, errorMessage)); } /** @@ -222,15 +279,6 @@ boolean checkRequestParameters(final RestRequest request, final RestChannel chan // error_trace cannot be used when we disable detailed errors // we consume the error_trace parameter first to ensure that it is always consumed if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) { - try { - XContentBuilder builder = channel.newErrorBuilder(); - builder.startObject().field("error", "error traces in responses are disabled.").endObject().string(); - RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); - response.addHeader("Content-Type", "application/json"); - channel.sendResponse(response); - } catch (IOException e) { - logger.warn("Failed to send response", e); - } return false; } @@ -312,8 +360,8 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - return delegate.newBuilder(autoDetectSource, useFiltering); + public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(xContentType, useFiltering); } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index 393e425baf937..740b98ae737e2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -28,7 +28,6 @@ public interface RestHandler { /** * Handles a rest request. - * * @param request The request to handle * @param channel The channel to write the request response to * @param client A client to use to make internal requests on behalf of the original request @@ -38,4 +37,13 @@ public interface RestHandler { default boolean canTripCircuitBreaker() { return true; } + + /** + * Indicates if a RestHandler supports plain text bodies + * @deprecated use request parameters or bodies that can be parsed with XContent! + */ + @Deprecated + default boolean supportsPlainText() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index b4b0614f8ad08..7d41faf12e074 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; @@ -26,20 +27,26 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue; @@ -47,12 +54,25 @@ public abstract class RestRequest implements ToXContent.Params { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class)); + // tchar pattern as defined by RFC7230 section 3.2.6 + private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); + private final NamedXContentRegistry xContentRegistry; private final Map params; + private final Map> headers; private final String rawPath; private final Set consumedParams = new HashSet<>(); + private final SetOnce xContentType = new SetOnce<>(); - public RestRequest(NamedXContentRegistry xContentRegistry, String uri) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param uri the URI of the request that potentially contains request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map> headers) { this.xContentRegistry = xContentRegistry; final Map params = new HashMap<>(); int pathEndPos = uri.indexOf('?'); @@ -63,12 +83,32 @@ public RestRequest(NamedXContentRegistry xContentRegistry, String uri) { RestUtils.decodeQueryString(uri, pathEndPos + 1, params); } this.params = params; + this.headers = Collections.unmodifiableMap(headers); + final List contentType = getAllHeaderValues("Content-Type"); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } - public RestRequest(NamedXContentRegistry xContentRegistry, Map params, String path) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param params the parameters of the request + * @param path the path of the request. This should not contain request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, Map params, String path, Map> headers) { this.xContentRegistry = xContentRegistry; this.params = params; this.rawPath = path; + this.headers = Collections.unmodifiableMap(headers); + final List contentType = getAllHeaderValues("Content-Type"); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } public enum Method { @@ -100,9 +140,53 @@ public final String path() { public abstract BytesReference content(); - public abstract String header(String name); + /** + * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are + * sent. Use of {@link #getAllHeaderValues(String)} should be preferred + */ + public final String header(String name) { + List values = headers.get(name); + if (values != null && values.isEmpty() == false) { + return values.get(0); + } + return null; + } + + /** + * Get all values for the header or {@code null} if the header was not found + */ + public final List getAllHeaderValues(String name) { + List values = headers.get(name); + if (values != null) { + return Collections.unmodifiableList(values); + } + return null; + } + + /** + * Get all of the headers and values associated with the headers. Modifications of this map are not supported. + */ + public final Map> getHeaders() { + return headers; + } - public abstract Iterable> headers(); + /** + * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of + * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request + */ + @Nullable + public final XContentType getXContentType() { + return xContentType.get(); + } + + /** + * Sets the {@link XContentType} + * @deprecated this is only used to allow BWC with content-type detection + */ + @Deprecated + final void setXContentType(XContentType xContentType) { + this.xContentType.set(xContentType); + } @Nullable public SocketAddress getRemoteAddress() { @@ -254,8 +338,10 @@ public final XContentParser contentParser() throws IOException { BytesReference content = content(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); + } else if (xContentType.get() == null) { + throw new IllegalStateException("unknown content type"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return xContentType.get().xContent().createParser(xContentRegistry, content); } /** @@ -283,11 +369,12 @@ public final boolean hasContentOrSourceParam() { * if you need to handle the absence request content gracefully. */ public final XContentParser contentOrSourceParamParser() throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return tuple.v1().xContent().createParser(xContentRegistry, content); } /** @@ -296,9 +383,11 @@ public final XContentParser contentOrSourceParamParser() throws IOException { * back to the user when there isn't request content. */ public final void withContentOrSourceParamParserOrNull(CheckedConsumer withParser) throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); if (content.length() > 0) { - try (XContentParser parser = XContentFactory.xContent(content).createParser(xContentRegistry, content)) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { withParser.accept(parser); } } else { @@ -310,7 +399,66 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { + if (hasContent()) { + if (xContentType.get() == null) { + throw new IllegalStateException("unknown content type"); + } + return new Tuple<>(xContentType.get(), content()); + } + + String source = param("source"); + String typeParam = param("source_content_type"); + if (source != null) { + BytesArray bytes = new BytesArray(source); + final XContentType xContentType; + if (typeParam != null) { + xContentType = parseContentType(Collections.singletonList(typeParam)); + } else { + DEPRECATION_LOGGER.deprecated("Deprecated use of the [source] parameter without the [source_content_type] parameter. Use " + + "the [source_content_type] parameter to specify the content type of the source such as [application/json]"); + xContentType = XContentFactory.xContentType(bytes); + } + + if (xContentType == null) { + throw new IllegalStateException("could not determine source content type"); + } + return new Tuple<>(xContentType, bytes); + } + return new Tuple<>(XContentType.JSON, BytesArray.EMPTY); + } + + /** + * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source} + * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception + * back to the user when there isn't request content. This version allows for plain text content + */ + @Deprecated + public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer withParser) + throws IOException { + if (hasContent() && xContentType.get() == null) { + withParser.accept(null); + } + + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); + if (content.length() > 0) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { + withParser.accept(parser); + } + } else { + withParser.accept(null); + } + } + + /** + * Get the content of the request or the contents of the {@code source} param without the xcontent type. This is useful the request can + * accept non xcontent values. + * @deprecated we should only take xcontent + */ + @Deprecated + public final BytesReference getContentOrSourceParamOnly() { if (hasContent()) { return content(); } @@ -320,4 +468,29 @@ public final BytesReference contentOrSourceParam() { } return BytesArray.EMPTY; } + + /** + * Parses the given content type string for the media type. This method currently ignores parameters. + */ + // TODO stop ignoring parameters such as charset... + private static XContentType parseContentType(List header) { + if (header == null || header.isEmpty()) { + return null; + } else if (header.size() > 1) { + throw new IllegalArgumentException("only one Content-Type header should be provided"); + } + + String rawContentType = header.get(0); + final String[] elements = rawContentType.split("[ \t]*;"); + if (elements.length > 0) { + final String[] splitMediaType = elements[0].split("/"); + if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() + && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { + return XContentType.fromMediaType(elements[0]); + } else { + throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); + } + } + throw new IllegalArgumentException("empty Content-Type header"); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index cee46eba04a5a..358c3656ceddf 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -65,7 +65,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client "specifying lang [" + lang + "] as part of the url path is deprecated, use request content instead"); } - PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content); + PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content, request.getXContentType()); return channel -> client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index ea0cf7bf0c76d..8c314efee634a 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -42,7 +42,7 @@ public RestCreateIndexAction(Settings settings, RestController controller) { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { - createIndexRequest.source(request.content()); + createIndexRequest.source(request.content(), request.getXContentType()); } createIndexRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index c48ed3716db5e..b376f3dab30a2 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -57,7 +57,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout())); putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - putRequest.source(request.content()); + putRequest.source(request.content(), request.getXContentType()); return channel -> client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<>(channel)); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 60c434c6b0ab8..06fe67926455f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; @@ -65,11 +64,11 @@ public RestPutMappingAction(Settings settings, RestController controller) { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); putMappingRequest.type(request.param("type")); - putMappingRequest.source(request.content().utf8ToString()); + putMappingRequest.source(request.content(), request.getXContentType()); putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); - return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener(channel)); + return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java index 9e38fb5e07411..47037460e2b6e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java @@ -66,7 +66,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Settings.Builder updateSettings = Settings.builder(); String bodySettingsStr = request.content().utf8ToString(); if (Strings.hasText(bodySettingsStr)) { - Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build(); + Settings buildSettings = Settings.builder() + .loadFromSource(bodySettingsStr, request.getXContentType()) + .build(); for (Map.Entry entry : buildSettings.getAsMap().entrySet()) { String key = entry.getKey(); String value = entry.getValue(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index af0faac553fc9..8fdf2792a4617 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -93,7 +93,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, - defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex); + defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestBuilderListener(channel) { @Override diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index 728e1ff5991cb..83d424ed74dc6 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -66,7 +66,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC client.get(getRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(GetResponse response) throws Exception { - XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false); + XContentBuilder builder = channel.newBuilder(request.getXContentType(), false); if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing return new BytesRestResponse(NOT_FOUND, builder); } else { diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 042b0d57b565b..ddaf22687563f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -65,7 +65,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC indexRequest.routing(request.param("routing")); indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing indexRequest.setPipeline(request.param("pipeline")); - indexRequest.source(request.content()); + indexRequest.source(request.content(), request.getXContentType()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java index b7a8e6366bbd4..2496c9b4a2487 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java @@ -21,7 +21,10 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -38,7 +41,8 @@ public RestPutPipelineAction(Settings settings, RestController controller) { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), restRequest.contentOrSourceParam()); + Tuple sourceTuple = restRequest.contentOrSourceParam(); + PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout())); request.timeout(restRequest.paramAsTime("timeout", request.timeout())); return channel -> client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java index 5885dc9df6054..9dbe1808a8c59 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -21,7 +21,10 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -40,7 +43,8 @@ public RestSimulatePipelineAction(Settings settings, RestController controller) @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - SimulatePipelineRequest request = new SimulatePipelineRequest(restRequest.contentOrSourceParam()); + Tuple sourceTuple = restRequest.contentOrSourceParam(); + SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1()); request.setId(restRequest.param("id")); request.setVerbose(restRequest.paramAsBoolean("verbose", false)); return channel -> client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java index e7ee7c73973e1..47252f5a10217 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java @@ -20,12 +20,10 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -50,26 +48,34 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC String scrollIds = request.param("scroll_id"); ClearScrollRequest clearRequest = new ClearScrollRequest(); clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - scrollIds = body.utf8ToString(); - clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); + request.withContentOrSourceParamParserOrNullLenient((xContentParser -> { + if (xContentParser == null) { + if (request.hasContent()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.content(); + String bodyScrollIds = body.utf8ToString(); + clearRequest.setScrollIds(Arrays.asList(splitScrollIds(bodyScrollIds))); + } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value clearRequest.setScrollIds(null); - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, clearRequest); + try { + buildFromContent(xContentParser, clearRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + })); - return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener(channel)); + return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel)); + } + + @Override + public boolean supportsPlainText() { + return true; } - public static String[] splitScrollIds(String scrollIds) { + private static String[] splitScrollIds(String scrollIds) { if (scrollIds == null) { return Strings.EMPTY_ARRAY; } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 567823a6e2d10..a803958618678 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -26,10 +26,11 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -93,7 +94,7 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a } /** - * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer. + * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer. */ public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex, BiConsumer consumer) throws IOException { @@ -103,9 +104,10 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind String searchType = request.param("search_type"); String routing = request.param("routing"); - final BytesReference data = request.contentOrSourceParam(); + final Tuple sourceTuple = request.contentOrSourceParam(); + final XContent xContent = sourceTuple.v1().xContent(); + final BytesReference data = sourceTuple.v2(); - XContent xContent = XContentFactory.xContent(data); int from = 0; int length = data.length(); byte marker = xContent.streamSeparator(); @@ -176,7 +178,7 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind break; } BytesReference bytes = data.slice(from, nextMarker - from); - try (XContentParser parser = XContentFactory.xContent(bytes).createParser(request.getXContentRegistry(), bytes)) { + try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) { consumer.accept(searchRequest, parser); } // move pointers diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index 87a6f464ccafa..2a60fc6317a3c 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -58,25 +57,33 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll"))); } - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - if (scrollId == null) { - scrollId = body.utf8ToString(); - searchScrollRequest.scrollId(scrollId); + request.withContentOrSourceParamParserOrNull(xContentParser -> { + if (xContentParser == null) { + if (request.hasContent()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.getContentOrSourceParamOnly(); + if (scrollId == null) { + String bodyScrollId = body.utf8ToString(); + searchScrollRequest.scrollId(bodyScrollId); + } } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, searchScrollRequest); + try { + buildFromContent(xContentParser, searchScrollRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + }); return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } + @Override + public boolean supportsPlainText() { + return true; + } + public static void buildFromContent(XContentParser parser, SearchScrollRequest searchScrollRequest) throws IOException { if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Malformed content, must start with an object"); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java index e9fb6e126fc2d..87afc21a8c020 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -276,7 +276,6 @@ public static ScriptMetaData fromXContent(XContentParser parser) throws IOExcept } else { source = new StoredScriptSource(id.substring(0, split), parser.text(), Collections.emptyMap()); } - scripts.put(id, source); id = null; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index f9bcc77ab4703..692e081a7ba1a 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -389,7 +389,7 @@ public void putStoredScript(ClusterService clusterService, PutStoredScriptReques request.content().length() + "] for script [" + request.id() + "]"); } - StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content()); + StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType()); if (isLangSupported(source.getLang()) == false) { throw new IllegalArgumentException("unable to put stored script with unsupported lang [" + source.getLang() + "]"); diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java index acf7135424f84..ab1d2f50bef43 100644 --- a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java +++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.UncheckedIOException; @@ -231,8 +232,8 @@ private StoredScriptSource build() { * @param content The content from the request to be parsed as described above. * @return The parsed {@link StoredScriptSource}. */ - public static StoredScriptSource parse(String lang, BytesReference content) { - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, content)) { + public static StoredScriptSource parse(String lang, BytesReference content, XContentType xContentType) { + try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) { Token token = parser.nextToken(); if (token != Token.START_OBJECT) { diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java index 791501373e6dd..cca9d73b78f16 100644 --- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java @@ -75,6 +75,7 @@ import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; @@ -203,7 +204,8 @@ public void testIndex() { String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"}; interceptTransportActions(indexShardActions); - IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id").source("field", "value"); + IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id") + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); internalCluster().coordOnlyNodeClient().index(indexRequest).actionGet(); clearInterceptedActions(); @@ -228,7 +230,7 @@ public void testUpdate() { String indexOrAlias = randomIndexOrAlias(); client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get(); - UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); @@ -242,7 +244,8 @@ public void testUpdateUpsert() { interceptTransportActions(updateShardActions); String indexOrAlias = randomIndexOrAlias(); - UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert("field", "value").doc("field1", "value1"); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert(Requests.INDEX_CONTENT_TYPE, "field", "value") + .doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet(); assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult()); @@ -275,7 +278,7 @@ public void testBulk() { int numIndexRequests = iterations(1, 10); for (int i = 0; i < numIndexRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value")); + bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); indices.add(indexOrAlias); } int numDeleteRequests = iterations(1, 10); @@ -287,7 +290,7 @@ public void testBulk() { int numUpdateRequests = iterations(1, 10); for (int i = 0; i < numUpdateRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1")); + bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1")); indices.add(indexOrAlias); } diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java index 60e4b4d26224f..d0e8d997a1bb9 100644 --- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java +++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.test.ESIntegTestCase; @@ -38,7 +39,7 @@ public void testThreadedListeners() throws Throwable { IndexRequest request = new IndexRequest("test", "type", "1"); if (randomBoolean()) { // set the source, without it, we will have a verification failure - request.source("field1", "value1"); + request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); } client.index(request, new ActionListener() { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 7f490ebab9050..580d8149fd5bc 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -300,7 +301,8 @@ public void testTransportBulkTasks() { registerTaskManageListeners(BulkAction.NAME + "[s][r]"); // shard task on replica createIndex("test"); ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks - client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}")).get(); + client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id") + .setSource("{\"foo\": \"bar\"}", XContentType.JSON)).get(); // the bulk operation should produce one main task List topTask = findEvents(BulkAction.NAME, Tuple::v1); @@ -352,7 +354,7 @@ public void testSearchTaskDescriptions() { registerTaskManageListeners(SearchAction.NAME + "[*]"); // shard task createIndex("test"); ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks - client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}") + client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}", XContentType.JSON) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertSearchResponse(client().prepareSearch("test").setTypes("doc").setQuery(QueryBuilders.matchAllQuery()).get()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index fbe0302e5d9d5..1e9256d20e3ff 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -801,6 +801,6 @@ private Map serialize(ListTasksResponse response, boolean byPare builder.endObject(); builder.flush(); logger.info(builder.string()); - return XContentHelper.convertToMap(builder.bytes(), false).v2(); + return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 546b1435c3513..4b531267d3ae9 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -209,7 +209,7 @@ private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOExcep FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry()); requestBuilder.withParams(params); if (hasBody) { - requestBuilder.withContent(builder.bytes()); + requestBuilder.withContent(builder.bytes(), builder.contentType()); } return requestBuilder.build(); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java new file mode 100644 index 0000000000000..5f3e107942178 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Base64; + +public class PutStoredScriptRequestTests extends ESTestCase { + + public void testSerialization() throws IOException { + PutStoredScriptRequest storedScriptRequest = new PutStoredScriptRequest("foo", "bar", new BytesArray("{}"), XContentType.JSON); + + assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + try (BytesStreamOutput output = new BytesStreamOutput()) { + storedScriptRequest.writeTo(output); + + try (StreamInput in = output.bytes().streamInput()) { + PutStoredScriptRequest serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals(storedScriptRequest.lang(), serialized.lang()); + assertEquals(storedScriptRequest.id(), serialized.id()); + } + } + } + + public void testSerializationBwc() throws IOException { + final byte[] rawStreamBytes = Base64.getDecoder().decode("ADwDCG11c3RhY2hlAQZzY3JpcHQCe30A"); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(rawStreamBytes)) { + in.setVersion(version); + PutStoredScriptRequest serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals("mustache", serialized.lang()); + assertEquals("script", serialized.id()); + assertEquals(new BytesArray("{}"), serialized.content()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + serialized.writeTo(out); + out.flush(); + assertArrayEquals(rawStreamBytes, out.bytes().toBytesRef().bytes); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java index ad17689bb789c..1ae60f42eb63d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.junit.After; @@ -57,7 +58,7 @@ public void tearDown() throws Exception { */ public void testSetSource() throws IOException { CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE); - builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}"); + builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); XContentBuilder xContent = XContentFactory.jsonBuilder().startObject().field(KEY, VALUE).endObject(); @@ -68,7 +69,7 @@ public void testSetSource() throws IOException { ByteArrayOutputStream docOut = new ByteArrayOutputStream(); XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field(KEY, VALUE).endObject(); doc.close(); - builder.setSource(docOut.toByteArray()); + builder.setSource(docOut.toByteArray(), XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); Map settingsMap = new HashMap<>(); @@ -85,7 +86,7 @@ public void testSetSettings() throws IOException { builder.setSettings(KEY, VALUE); assertEquals(VALUE, builder.request().settings().get(KEY)); - builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}"); + builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); builder.setSettings(Settings.builder().put(KEY, VALUE)); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java new file mode 100644 index 0000000000000..590eba3666610 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Base64; + +public class CreateIndexRequestTests extends ESTestCase { + + public void testSerialization() throws IOException { + CreateIndexRequest request = new CreateIndexRequest("foo"); + String mapping = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().string(); + request.mapping("my_type", mapping, XContentType.JSON); + + try (BytesStreamOutput output = new BytesStreamOutput()) { + request.writeTo(output); + + try (StreamInput in = output.bytes().streamInput()) { + CreateIndexRequest serialized = new CreateIndexRequest(); + serialized.readFrom(in); + assertEquals(request.index(), serialized.index()); + assertEquals(mapping, serialized.mappings().get("my_type")); + } + } + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAANmb28APAMBB215X3R5cGULeyJ0eXBlIjp7fX0AAAD////+AA=="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + CreateIndexRequest serialized = new CreateIndexRequest(); + serialized.readFrom(in); + assertEquals("foo", serialized.index()); + BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); + assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type")); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + serialized.writeTo(out); + out.flush(); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index e19f930a22d22..9ab6551d6fd07 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -59,7 +60,8 @@ public void testCreateShrinkIndexToN() { internalCluster().ensureAtLeastNumDataNodes(2); prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); @@ -85,7 +87,8 @@ public void testCreateShrinkIndexToN() { assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); for (int i = 0; i < 20; i++) { // now update - client().prepareIndex("first_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("first_shrink", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); @@ -113,7 +116,8 @@ public void testCreateShrinkIndexToN() { assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); for (int i = 0; i < 20; i++) { // now update - client().prepareIndex("second_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("second_shrink", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); @@ -129,7 +133,8 @@ public void testCreateShrinkIndex() { .put("index.version.created", version) ).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); @@ -164,7 +169,8 @@ public void testCreateShrinkIndex() { } for (int i = 20; i < 40; i++) { - client().prepareIndex("target", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("target", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40); @@ -181,7 +187,8 @@ public void testCreateShrinkIndexFails() throws Exception { .put("number_of_shards", randomIntBetween(2, 7)) .put("number_of_replicas", 0)).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java new file mode 100644 index 0000000000000..4dc396323c048 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.mapping.get; + +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class GetFieldMappingsResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + Map>> mappings = new HashMap<>(); + FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}")); + mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData))); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + GetFieldMappingsResponse serialized = new GetFieldMappingsResponse(); + try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) { + serialized.readFrom(in); + FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field"); + assertNotNull(metaData); + assertEquals(new BytesArray("{}"), metaData.getSource()); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 9c93e5c73d9f5..fd7f830e59230 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -19,10 +19,20 @@ package org.elasticsearch.action.admin.indices.mapping.put; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; +import java.util.Base64; + public class PutMappingRequestTests extends ESTestCase { public void testValidation() { @@ -41,12 +51,12 @@ public void testValidation() { assertNotNull("source validation should fail", ex); assertTrue(ex.getMessage().contains("source is missing")); - r.source(""); + r.source("", XContentType.JSON); ex = r.validate(); assertNotNull("source validation should fail", ex); assertTrue(ex.getMessage().contains("source is empty")); - r.source("somevalidmapping"); + r.source("somevalidmapping", XContentType.JSON); ex = r.validate(); assertNull("validation should succeed", ex); @@ -64,4 +74,33 @@ public void testBuildFromSimplifiedDef() { () -> PutMappingRequest.buildFromSimplifiedDef("type", "only_field")); assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage()); } + + public void testPutMappingRequestSerialization() throws IOException { + PutMappingRequest request = new PutMappingRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.source(mapping, XContentType.YAML); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source()); + + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + request.writeTo(bytesStreamOutput); + StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); + PutMappingRequest serialized = new PutMappingRequest(); + serialized.readFrom(in); + + String source = serialized.source(); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source); + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAQNmb28MAA8tLS0KZm9vOiAiYmFyIgoAPAMAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutMappingRequest request = new PutMappingRequest(); + request.readFrom(in); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source()); + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java index 7ea103313fe83..3615ea10c361c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java @@ -19,9 +19,10 @@ package org.elasticsearch.action.admin.indices.template; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESSingleNodeTestCase; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; /** * Rudimentary tests that the templates used by Logstash and Beats @@ -29,14 +30,14 @@ */ public class BWCTemplateTests extends ESSingleNodeTestCase { public void testBeatsTemplatesBWC() throws Exception { - String metricBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json"); - String packetBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json"); - String fileBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json"); - String winLogBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json"); - client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat).get(); - client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat).get(); - client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat).get(); - client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat).get(); + byte[] metricBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json"); + byte[] packetBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json"); + byte[] fileBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json"); + byte[] winLogBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json"); + client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat, XContentType.JSON).get(); client().prepareIndex("metricbeat-foo", "doc", "1").setSource("message", "foo").get(); client().prepareIndex("packetbeat-foo", "doc", "1").setSource("message", "foo").get(); @@ -46,8 +47,8 @@ public void testBeatsTemplatesBWC() throws Exception { } public void testLogstashTemplatesBWC() throws Exception { - String ls5x = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json"); - client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x).get(); + byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json"); + client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get(); client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get(); assertWarnings("Deprecated field [template] used, replaced by [index_patterns]"); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index 076245ad76d5b..48b2ae79cf39c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -22,6 +22,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -66,4 +69,36 @@ public void testPutIndexTemplateRequest510() throws IOException { } } + public void testPutIndexTemplateRequestSerializationXContent() throws IOException { + PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.patterns(Collections.singletonList("foo")); + request.mapping("bar", mapping, XContentType.YAML); + assertNotEquals(mapping, request.mappings().get("bar")); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar")); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); + serialized.readFrom(in); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), serialized.mappings().get("bar")); + } + + public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAANmb28IdGVtcGxhdGUAAAAAAAABA2Jhcg8tLS0KZm9vOiAiYmFyIgoAAAAAAAAAAAAAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.readFrom(in); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + assertNotEquals(mapping, request.mappings().get("bar")); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar")); + assertEquals("foo", request.name()); + assertEquals("template", request.patterns().get(0)); + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 4300a629fbd8c..22377ea1769c8 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import java.nio.charset.StandardCharsets; @@ -31,7 +32,7 @@ public class BulkIntegrationIT extends ESIntegTestCase { public void testBulkIndexCreatesMapping() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/bulk-log.json"); BulkRequestBuilder bulkBuilder = client().prepareBulk(); - bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); bulkBuilder.get(); assertBusy(new Runnable() { @Override diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index 48d989209cbba..ca1630b00641c 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -36,9 +37,9 @@ public void testBulkProcessorAutoCreateRestrictions() throws Exception { client().admin().cluster().prepareHealth("willwork").setWaitForGreenStatus().execute().actionGet(); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}")); - bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}")); - bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}")); + bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}", XContentType.JSON)); BulkResponse br = bulkRequestBuilder.get(); BulkItemResponse[] responses = br.getItems(); assertEquals(3, responses.length); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java index 6bac7c2f8a436..7e608815c43c4 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -254,11 +255,13 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception for (int i = 1; i <= numDocs; i++) { if (randomBoolean()) { testDocs++; - processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source("field", "value")); + processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")); multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs)); } else { testReadOnlyDocs++; - processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source("field", "value")); + processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")); } } } @@ -296,7 +299,8 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) { MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet(); for (int i = 1; i <= numDocs; i++) { - processor.add(new IndexRequest("test", "test", Integer.toString(i)).source("field", randomRealisticUnicodeOfLengthBetween(1, 30))); + processor.add(new IndexRequest("test", "test", Integer.toString(i)) + .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30))); multiGetRequestBuilder.add("test", "test", Integer.toString(i)); } return multiGetRequestBuilder; diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java index 4a41499d0d127..b6242e6d5fcd4 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -44,7 +45,7 @@ public void testBulkRequestModifier() { int numRequests = scaledRandomIntBetween(8, 64); BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { - bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}")); + bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}", XContentType.JSON)); } CaptureActionListener actionListener = new CaptureActionListener(); TransportBulkAction.BulkRequestModifier bulkRequestModifier = new TransportBulkAction.BulkRequestModifier(bulkRequest); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index e0c58a60d297f..a35a82ff096eb 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -61,7 +61,7 @@ public void testSimpleBulk1() throws Exception { bulkAction = Strings.replace(bulkAction, "\r\n", "\n"); } BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }"))); assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class)); @@ -71,21 +71,21 @@ public void testSimpleBulk1() throws Exception { public void testSimpleBulk2() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } public void testSimpleBulk3() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk3.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } public void testSimpleBulk4() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk4.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(4)); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1")); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2)); @@ -107,21 +107,21 @@ public void testSimpleBulk4() throws Exception { public void testBulkAllowExplicitIndex() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); try { - new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false); + new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false, XContentType.JSON); fail(); } catch (Exception e) { } bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk5.json"); - new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false); + new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false, XContentType.JSON); } public void testBulkAddIterable() { BulkRequest bulkRequest = Requests.bulkRequest(); List requests = new ArrayList<>(); - requests.add(new IndexRequest("test", "test", "id").source("field", "value")); - requests.add(new UpdateRequest("test", "test", "id").doc("field", "value")); + requests.add(new IndexRequest("test", "test", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); + requests.add(new UpdateRequest("test", "test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")); requests.add(new DeleteRequest("test", "test", "id")); bulkRequest.add(requests); assertThat(bulkRequest.requests().size(), equalTo(3)); @@ -134,7 +134,7 @@ public void testSimpleBulk6() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk6.json"); BulkRequest bulkRequest = new BulkRequest(); ParsingException exc = expectThrows(ParsingException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Unknown key for a VALUE_STRING in [hello]")); } @@ -142,7 +142,7 @@ public void testSimpleBulk7() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk7.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Malformed action/metadata line [5], expected a simple value for field [_unkown] but found [START_ARRAY]")); } @@ -151,7 +151,7 @@ public void testSimpleBulk8() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk8.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Action/metadata line [3] contains an unknown parameter [_foo]")); } @@ -159,14 +159,14 @@ public void testSimpleBulk9() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk9.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]")); } public void testSimpleBulk10() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk10.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(9)); } @@ -178,8 +178,8 @@ public void testBulkRequestWithRefresh() throws Exception { // We force here a "type is missing" validation error bulkRequest.add(new DeleteRequest("index", null, "id")); bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new IndexRequest("index", "type", "id").source("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); ActionRequestValidationException validate = bulkRequest.validate(); assertThat(validate, notNullValue()); assertThat(validate.validationErrors(), not(empty())); @@ -202,7 +202,8 @@ public void testBulkNoSource() throws Exception { assertThat(validate.validationErrors(), not(empty())); assertThat(validate.validationErrors(), contains( "script or doc is missing", - "source is missing")); + "source is missing", + "content type is missing")); } public void testCannotAddNullRequests() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 82eee3554e804..b31426e3978dd 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -29,8 +29,10 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -209,9 +211,9 @@ public void testBulkVersioning() throws Exception { assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L)); bulkResponse = client().prepareBulk() - .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc("field", "2")) - .add(client().prepareUpdate("test", "type", "2").setDoc("field", "2")) - .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc("field", "3")).get(); + .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2")) + .add(client().prepareUpdate("test", "type", "2").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2")) + .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3")).get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L)); @@ -235,9 +237,9 @@ public void testBulkVersioning() throws Exception { bulkResponse = client().prepareBulk() .add(client().prepareUpdate("test", "type", "e1") - .setDoc("field", "2").setVersion(10)) // INTERNAL + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2").setVersion(10)) // INTERNAL .add(client().prepareUpdate("test", "type", "e1") - .setDoc("field", "3").setVersion(13).setVersionType(VersionType.INTERNAL)) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3").setVersion(13).setVersionType(VersionType.INTERNAL)) .get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); @@ -493,8 +495,8 @@ public void testBulkUpdateDocAsUpsertWithParent() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChild, 0, addChild.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChild, 0, addChild.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(2)); @@ -576,9 +578,9 @@ public void testBulkUpdateUpsertWithParent() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChild1, 0, addChild1.length); - builder.add(addChild2, 0, addChild2.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChild1, 0, addChild1.length, XContentType.JSON); + builder.add(addChild2, 0, addChild2.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -650,10 +652,10 @@ public void testBulkUpdateChildMissingParentRouting() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChildOK, 0, addChildOK.length); - builder.add(addChildMissingRouting, 0, addChildMissingRouting.length); - builder.add(addChildOK, 0, addChildOK.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON); + builder.add(addChildMissingRouting, 0, addChildMissingRouting.length, XContentType.JSON); + builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(4)); @@ -680,7 +682,8 @@ public void testFailingVersionedUpdatedOnBulk() throws Exception { return; } BulkRequestBuilder requestBuilder = client().prepareBulk(); - requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID)); + requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID)); responses[threadID] = requestBuilder.get(); }); @@ -733,19 +736,19 @@ public void testThatInvalidIndexNamesShouldNotBreakCompleteBulkRequest() { // issue 6630 public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { BulkResponse indexBulkItemResponse = client().prepareBulk() - .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }")) - .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }")) - .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }")) - .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }")) + .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }", XContentType.JSON)) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); assertNoFailures(indexBulkItemResponse); BulkResponse bulkItemResponse = client().prepareBulk() - .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }")) - .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }")) - .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}")) - .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}")) + .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }", XContentType.JSON)) + .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}", XContentType.JSON)) + .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}", XContentType.JSON)) .add(new DeleteRequest("test", "type", "5")) .add(new DeleteRequest("test", "type", "6")) .get(); @@ -768,10 +771,10 @@ private static String indexOrAlias() { public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{ createIndex("bulkindex1", "bulkindex2"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1")) - .add(new IndexRequest("bulkindex2", "index2_type", "1").source("text", "hallo2")) - .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2")) - .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar")) + bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) + .add(new IndexRequest("bulkindex2", "index2_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) + .add(new IndexRequest("bulkindex2", "index2_type").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) + .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex2", "index2_type", "3")) .setRefreshPolicy(RefreshPolicy.IMMEDIATE); @@ -794,8 +797,8 @@ public void testFailedRequestsOnClosedIndex() throws Exception { assertAcked(client().admin().indices().prepareClose("bulkindex1")); BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE); - bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1")) - .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar")) + bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) + .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex1", "index1_type", "1")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); @@ -810,8 +813,10 @@ public void testFailedRequestsOnClosedIndex() throws Exception { // issue 9821 public void testInvalidIndexNamesCorrectOpType() { BulkResponse bulkResponse = client().prepareBulk() - .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1").setSource("field", 1)) - .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1").setDoc("field", randomInt())) + .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1") + .setSource(Requests.INDEX_CONTENT_TYPE, "field", 1)) + .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt())) .add(client().prepareDelete().setIndex("INVALID.NAME").setType("type1").setId("1")).get(); assertThat(bulkResponse.getItems().length, is(3)); assertThat(bulkResponse.getItems()[0].getOpType(), is(OpType.INDEX)); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 29c55c426d313..cdfe54575fa9b 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; @@ -174,7 +175,7 @@ private void runTestTook(boolean controlled) throws Exception { bulkAction = Strings.replace(bulkAction, "\r\n", "\n"); } BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); AtomicLong expected = new AtomicLong(); TransportBulkAction action = createAction(controlled, expected); action.doExecute(null, bulkRequest, new ActionListener() { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java index 4b4f24f5b8fee..b74056341136c 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.shard.ShardId; @@ -388,7 +389,7 @@ protected AbstractAsyncBulkByScrollAction.RequestWrapper buildRequest(Hit doc } }; ScrollableHitSource.BasicHit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0); - hit.setSource(new BytesArray("{}")); + hit.setSource(new BytesArray("{}"), XContentType.JSON); ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null); simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response); ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get()); diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java index 06e9d586e3671..5341588f7226b 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java @@ -74,8 +74,9 @@ public void testSetSource() throws Exception { ByteArrayOutputStream docOut = new ByteArrayOutputStream(); XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field("SomeKey", "SomeValue").endObject(); doc.close(); - indexRequestBuilder.setSource(docOut.toByteArray()); - assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true)); + indexRequestBuilder.setSource(docOut.toByteArray(), XContentType.JSON); + assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true, + indexRequestBuilder.request().getContentType())); doc = XContentFactory.jsonBuilder().startObject().field("SomeKey", "SomeValue").endObject(); doc.close(); diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index a722529626843..fb1991bcd91ea 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -18,17 +18,24 @@ */ package org.elasticsearch.action.index; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.HashSet; import java.util.Set; @@ -83,19 +90,19 @@ public void testCreateOperationRejectsVersions() { public void testIndexingRejectsLongIds() { String id = randomAsciiOfLength(511); IndexRequest request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); ActionRequestValidationException validate = request.validate(); assertNull(validate); id = randomAsciiOfLength(512); request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); validate = request.validate(); assertNull(validate); id = randomAsciiOfLength(513); request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); validate = request.validate(); assertThat(validate, notNullValue()); assertThat(validate.getMessage(), @@ -150,4 +157,49 @@ public void testIndexResponse() { ",shards={\"total\":" + total + ",\"successful\":" + successful + ",\"failed\":0}]", indexResponse.toString()); } + + public void testIndexRequestXContentSerialization() throws IOException { + IndexRequest indexRequest = new IndexRequest("foo", "bar", "1"); + indexRequest.source("{}", XContentType.JSON); + assertEquals(XContentType.JSON, indexRequest.getContentType()); + + BytesStreamOutput out = new BytesStreamOutput(); + indexRequest.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getContentType()); + assertEquals(new BytesArray("{}"), serialized.source()); + } + + public void testIndexRequestXContentSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAD////+AgQDZm9vAAAAAQNiYXIBATEAAAAAAnt9AP/////////9AAAA//////////8AAAAAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getContentType()); + assertEquals("{}", serialized.source().utf8ToString()); + // don't test writing to earlier versions since output differs due to no timestamp + } + } + + // reindex makes use of index requests without a source so this needs to be handled + public void testSerializationOfEmptyRequestWorks() throws IOException { + IndexRequest request = new IndexRequest("index", "type"); + assertNull(request.getContentType()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + + try (StreamInput in = out.bytes().streamInput()) { + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertNull(request.getContentType()); + assertEquals("index", request.index()); + assertEquals("type", request.type()); + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java new file mode 100644 index 0000000000000..01aed87947acf --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.ingest; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class PutPipelineRequestTests extends ESTestCase { + + public void testSerializationWithXContent() throws IOException { + PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, request.getXContentType()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + PutPipelineRequest serialized = new PutPipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDATECe30="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutPipelineRequest request = new PutPipelineRequest(); + request.readFrom(in); + assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals("{}", request.getSource().utf8ToString()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java index 0966010a8f79a..86dc56cdd04c0 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java @@ -19,19 +19,23 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.hamcrest.CoreMatchers.equalTo; public class SimulatePipelineRequestTests extends ESTestCase { public void testSerialization() throws IOException { - SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray("")); + SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""), XContentType.JSON); // Sometimes we set an id if (randomBoolean()) { request.setId(randomAsciiOfLengthBetween(1, 10)); @@ -51,4 +55,38 @@ public void testSerialization() throws IOException { assertThat(otherRequest.getId(), equalTo(request.getId())); assertThat(otherRequest.isVerbose(), equalTo(request.isVerbose())); } + + public void testSerializationWithXContent() throws IOException { + SimulatePipelineRequest request = + new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, request.getXContentType()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + SimulatePipelineRequest serialized = new SimulatePipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + } + + public void testSerializationWithXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAAAAnt9AAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + SimulatePipelineRequest request = new SimulatePipelineRequest(); + request.readFrom(in); + assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals("{}", request.getSource().utf8ToString()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index bc72094558e7c..d5417526c0cae 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -127,7 +127,7 @@ public void testToXContent() throws IOException { builder.startObject(); writeableIngestDocument.toXContent(builder, EMPTY_PARAMS); builder.endObject(); - Map toXContentMap = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map toXContentMap = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); Map toXContentDoc = (Map) toXContentMap.get("doc"); Map toXContentSource = (Map) toXContentDoc.get("_source"); diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index d249c33edcc6f..d1de2f7ccf756 100644 --- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; @@ -160,7 +161,8 @@ public void testMaxConcurrentSearchRequests() { private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException { byte[] data = StreamsUtils.copyToBytesFromClasspath(sample); - RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(data)).build(); + RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray(data), XContentType.JSON).build(); return RestMultiSearchAction.parseRequest(restRequest, true); } diff --git a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java index 2e1a00afc2048..cab27d74c7d58 100644 --- a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java +++ b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -44,9 +45,9 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertAcked(createIndexResponse); // indexing, by default, will work (waiting for one shard copy only) - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON).execute().actionGet(); try { - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(2) // wait for 2 active shard copies .setTimeout(timeValueMillis(100)).execute().actionGet(); fail("can't index, does not enough active shard copies"); @@ -70,12 +71,12 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // this should work, since we now have two - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(2) .setTimeout(timeValueSeconds(1)).execute().actionGet(); try { - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) .setTimeout(timeValueMillis(100)).execute().actionGet(); fail("can't index, not enough active shard copies"); @@ -92,7 +93,7 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); // this should work, since we now have all shards started - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) .setTimeout(timeValueSeconds(1)).execute().actionGet(); } diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java index c255ef6fc884f..e034cff3f1de0 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java @@ -36,12 +36,16 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -56,6 +60,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.EnumSet; import java.util.HashSet; import java.util.Set; @@ -236,6 +241,7 @@ public void testStreamRequest() throws IOException { request.parent(parent); String pref = random().nextBoolean() ? "somePreference" : null; request.preference(pref); + request.doc(new BytesArray("{}"), randomBoolean(), XContentType.JSON); // write ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); @@ -255,7 +261,37 @@ public void testStreamRequest() throws IOException { assertThat(request.termStatistics(), equalTo(req2.termStatistics())); assertThat(request.preference(), equalTo(pref)); assertThat(request.routing(), equalTo(null)); + assertEquals(new BytesArray("{}"), request.doc()); + assertEquals(XContentType.JSON, request.xContentType()); + } + } + public void testStreamRequestWithXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAABBWluZGV4BHR5cGUCaWQBAnt9AAABDnNvbWVQcmVmZXJlbmNlFgAAAAEA//////////0AAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + TermVectorsRequest request = new TermVectorsRequest(); + request.readFrom(in); + assertEquals("index", request.index()); + assertEquals("type", request.type()); + assertEquals("id", request.id()); + assertTrue(request.offsets()); + assertFalse(request.fieldStatistics()); + assertTrue(request.payloads()); + assertFalse(request.positions()); + assertTrue(request.termStatistics()); + assertNull(request.parent()); + assertEquals("somePreference", request.preference()); + assertEquals("{}", request.doc().utf8ToString()); + assertEquals(XContentType.JSON, request.xContentType()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } } } diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 892401356c34e..e1133542e4102 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -144,7 +144,8 @@ public void testUpdateRequest() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); - Map upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2(); + Map upsertDoc = + XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); @@ -171,7 +172,7 @@ public void testUpdateRequest() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); - upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2(); + upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 9f04f74cd2480..497b134ebe386 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; @@ -90,7 +91,8 @@ public void testAliases() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1")); logger.info("--> indexing against [alias1], should work now"); - IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet(); + IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getIndex(), equalTo("test")); logger.info("--> creating index [test_x]"); @@ -102,7 +104,8 @@ public void testAliases() throws Exception { assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1")); logger.info("--> indexing against [alias1], should work against [test_x]"); - indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet(); + indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getIndex(), equalTo("test_x")); } @@ -164,15 +167,14 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test"))); logger.info("--> indexing against [test]"); - client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index( - indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); + client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); logger.info("--> checking single filtering alias search"); SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get(); @@ -254,16 +256,16 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test2", "foos", termQuery("name", "foo"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get(); - client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get(); - client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get(); - client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get(); + client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); logger.info("--> indexing against [test2]"); - client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get(); - client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get(); - client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get(); - client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get(); + client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get(); refresh(); @@ -322,17 +324,17 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test3", "filter13", termQuery("name", "baz"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"))).get(); - client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"))).get(); - client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"))).get(); + client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"), XContentType.JSON)).get(); - client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"))).get(); - client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"))).get(); - client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"))).get(); + client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"), XContentType.JSON)).get(); - client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"))).get(); - client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"))).get(); - client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"))).get(); + client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"), XContentType.JSON)).get(); + client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"), XContentType.JSON)).get(); + client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"), XContentType.JSON)).get(); refresh(); @@ -382,16 +384,16 @@ public void testDeletingByQueryFilteringAliases() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test2", "tests", termQuery("name", "test"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get(); - client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get(); - client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get(); - client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get(); + client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); logger.info("--> indexing against [test2]"); - client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get(); - client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get(); - client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get(); - client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get(); + client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get(); refresh(); @@ -435,7 +437,7 @@ public void testWaitForAliasCreationMultipleShards() throws Exception { for (int i = 0; i < 10; i++) { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i)); - client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get(); + client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get(); } } @@ -447,7 +449,7 @@ public void testWaitForAliasCreationSingleShard() throws Exception { for (int i = 0; i < 10; i++) { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i)); - client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get(); + client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get(); } } @@ -466,7 +468,8 @@ public void testWaitForAliasSimultaneousUpdate() throws Exception { @Override public void run() { assertAcked(admin().indices().prepareAliases().addAlias("test", aliasName)); - client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"))).actionGet(); + client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"), XContentType.JSON)) + .actionGet(); } }); } @@ -745,7 +748,7 @@ public void testCreateIndexWithAliasesInSource() throws Exception { " \"alias2\" : {\"filter\" : {\"match_all\": {}}},\n" + " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" + " }\n" + - "}")); + "}", XContentType.JSON)); checkAliases(); } @@ -826,8 +829,8 @@ public void testAliasesFilterWithHasChildQuery() throws Exception { .addMapping("parent") .addMapping("child", "_parent", "type=parent") ); - client().prepareIndex("my-index", "parent", "1").setSource("{}").get(); - client().prepareIndex("my-index", "child", "2").setSource("{}").setParent("1").get(); + client().prepareIndex("my-index", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("my-index", "child", "2").setSource("{}", XContentType.JSON).setParent("1").get(); refresh(); assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", hasChildQuery("child", matchAllQuery(), ScoreMode.None))); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java index 20add0530d828..76e9a4eb1053c 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.RecoverySource; @@ -388,7 +389,7 @@ void assertRealtimeGetWorks(String indexName) { SearchHit hit = searchReq.get().getHits().getAt(0); String docId = hit.getId(); // foo is new, it is not a field in the generated index - client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get(); + client().prepareUpdate(indexName, "doc", docId).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "bar").get(); GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get(); Map source = getRsp.getSourceAsMap(); assertThat(source, Matchers.hasKey("foo")); diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java index ccc72db9d7a5f..d3a0e0b3e4e14 100644 --- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java +++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -106,7 +107,7 @@ public void testActions() { client.prepareSearch().execute().addListener(new AssertingActionListener<>(SearchAction.NAME, client.threadPool())); client.prepareDelete("idx", "type", "id").execute().addListener(new AssertingActionListener<>(DeleteAction.NAME, client.threadPool())); client.admin().cluster().prepareDeleteStoredScript("lang", "id").execute().addListener(new AssertingActionListener<>(DeleteStoredScriptAction.NAME, client.threadPool())); - client.prepareIndex("idx", "type", "id").setSource("source").execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool())); + client.prepareIndex("idx", "type", "id").setSource("source", XContentType.JSON).execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool())); // choosing arbitrary cluster admin actions to test client.admin().cluster().prepareClusterStats().execute().addListener(new AssertingActionListener<>(ClusterStatsAction.NAME, client.threadPool())); diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 73598c3effe4a..c799969a3c8e5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -236,7 +237,7 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { TimeValue timeout = TimeValue.timeValueMillis(200); long now = System.currentTimeMillis(); try { - client().prepareUpdate("test1", "type1", "1").setDoc("field", "value2").setTimeout(timeout).get(); + client().prepareUpdate("test1", "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout).get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index a0a0025681779..c2f6c3b64faae 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -40,7 +41,8 @@ public void testDataNodes() throws Exception { internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build()); client().admin().indices().create(createIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet(); try { - client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet(); + client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON) + .timeout(timeValueSeconds(1))).actionGet(); fail("no allocation should happen"); } catch (UnavailableShardsException e) { // all is well @@ -51,7 +53,8 @@ public void testDataNodes() throws Exception { // still no shard should be allocated try { - client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet(); + client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON) + .timeout(timeValueSeconds(1))).actionGet(); fail("no allocation should happen"); } catch (UnavailableShardsException e) { // all is well @@ -61,7 +64,8 @@ public void testDataNodes() throws Exception { internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build()); assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).execute().actionGet().isTimedOut(), equalTo(false)); - IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet(); + IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getId(), equalTo("1")); assertThat(indexResponse.getType(), equalTo("type1")); } diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java index 14e93a3bbc35b..93f745d290e5c 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java @@ -118,7 +118,7 @@ private void testFiltering(Settings source, Settings filtered, String... pattern source.toXContent(xContentBuilder, request); xContentBuilder.endObject(); String filteredSettingsString = xContentBuilder.string(); - filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build(); + filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build(); assertThat(filteredSettings.getAsMap().entrySet(), equalTo(filtered.getAsMap().entrySet())); } } diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java index d917c0d12c074..fc1300d94138e 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.CoreMatchers.containsString; @@ -52,7 +53,8 @@ public void testDuplicateKeysThrowsException() { assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", XContent.isStrictDuplicateDetectionEnabled()); final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}"; - final SettingsException e = expectThrows(SettingsException.class, () -> Settings.builder().loadFromSource(json).build()); + final SettingsException e = expectThrows(SettingsException.class, + () -> Settings.builder().loadFromSource(json, XContentType.JSON).build()); assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); assertThat( e.toString(), diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java index eeb2df7e1d68c..e4b4de0ceb616 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.nio.charset.StandardCharsets; @@ -74,7 +75,7 @@ public void testDuplicateKeysThrowsException() { String yaml = "foo: bar\nfoo: baz"; SettingsException e = expectThrows(SettingsException.class, () -> { - Settings.builder().loadFromSource(yaml); + Settings.builder().loadFromSource(yaml, XContentType.YAML); }); assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); String msg = e.getCause().getMessage(); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index a607b48f79ffd..3a1144afe93e0 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -385,7 +385,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exceptio .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj"}, Strings.EMPTY_ARRAY); assertThat(mapTuple.v2(), equalTo(filteredSource)); @@ -397,7 +397,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingExcludes() throws Exceptio .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"}); assertThat(mapTuple.v2(), equalTo(filteredSource)); @@ -410,7 +410,7 @@ public void testNotOmittingObjectsWithExcludedProperties() throws Exception { .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"obj.f1"}); assertThat(filteredSource.size(), equalTo(1)); @@ -430,7 +430,7 @@ public void testNotOmittingObjectWithNestedExcludedObject() throws Exception { .endObject(); // implicit include - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"*.obj2"}); assertThat(filteredSource.size(), equalTo(1)); @@ -460,7 +460,7 @@ public void testIncludingObjectWithNestedIncludedObject() throws Exception { .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, Strings.EMPTY_ARRAY); assertThat(filteredSource.size(), equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index b69b73da3a7c3..5e2869a658eba 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.FaultDetection; import org.elasticsearch.discovery.zen.MembershipAction; @@ -530,7 +531,7 @@ public void testAckedIndexing() throws Exception { int shard = Math.floorMod(Murmur3HashFunction.hash(id), numPrimaries); logger.trace("[{}] indexing id [{}] through node [{}] targeting shard [{}]", name, id, node, shard); IndexResponse response = - client.prepareIndex("test", "type", id).setSource("{}").setTimeout(timeout).get(timeout); + client.prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).setTimeout(timeout).get(timeout); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); ackedDocs.put(id, node); logger.trace("[{}] indexed id [{}] through node [{}]", name, id, node); @@ -1122,7 +1123,8 @@ public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Excep final String node_2 = internalCluster().startDataOnlyNode(); List indexRequestBuilderList = new ArrayList<>(); for (int i = 0; i < 100; i++) { - indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc").setSource("{\"int_field\":1}")); + indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc") + .setSource("{\"int_field\":1}", XContentType.JSON)); } indexRandom(true, indexRequestBuilderList); SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_2, random()); diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java index e3556c8cc7c63..34b4bc05727e3 100644 --- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -185,7 +186,7 @@ public void testBulk() throws Exception { .add(client().prepareIndex().setIndex("test").setType("type1").setId("2").setSource(source("2", "test")).setCreate(true)) .add(client().prepareIndex().setIndex("test").setType("type1").setSource(source("3", "test"))) .add(client().prepareDelete().setIndex("test").setType("type1").setId("1")) - .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }")) // failure + .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }", XContentType.JSON)) // failure .execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java index 814a861139e0a..84166bb3f962e 100644 --- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java +++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -45,7 +46,7 @@ public class ShardInfoIT extends ESIntegTestCase { public void testIndexAndDelete() throws Exception { prepareIndex(1); - IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get(); + IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON).get(); assertShardInfo(indexResponse); DeleteResponse deleteResponse = client().prepareDelete("idx", "type", indexResponse.getId()).get(); assertShardInfo(deleteResponse); @@ -53,7 +54,8 @@ public void testIndexAndDelete() throws Exception { public void testUpdate() throws Exception { prepareIndex(1); - UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}").setDocAsUpsert(true).get(); + UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true) + .get(); assertShardInfo(updateResponse); } @@ -61,7 +63,7 @@ public void testBulkWithIndexAndDeleteItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { - bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}")); + bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); } BulkResponse bulkResponse = bulkRequestBuilder.get(); @@ -83,7 +85,8 @@ public void testBulkWithUpdateItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { - bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}").setDocAsUpsert(true)); + bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}", XContentType.JSON) + .setDocAsUpsert(true)); } BulkResponse bulkResponse = bulkRequestBuilder.get(); diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index f6164d2f3448e..577087ba9522b 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.test.ESIntegTestCase; @@ -629,8 +630,8 @@ public void testGetFieldsComplexField() throws Exception { logger.info("indexing documents"); - client().prepareIndex("my-index", "my-type1", "1").setSource(source).get(); - client().prepareIndex("my-index", "my-type2", "1").setSource(source).get(); + client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get(); + client().prepareIndex("my-index", "my-type2", "1").setSource(source, XContentType.JSON).get(); logger.info("checking real time retrieval"); @@ -698,7 +699,7 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException { " }\n" + " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"suggest\": {\n" + @@ -737,10 +738,10 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { " }\n" + " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); - client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").get(); + client().prepareIndex("test", "doc").setId("1").setSource("{}", XContentType.JSON).setParent("1").get(); String[] fieldsList = {"_parent"}; // before refresh - document is only in translog @@ -761,12 +762,12 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"text\": \"some text.\"\n" + "}\n"; - client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); + client().prepareIndex("test", "doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get(); String[] fieldsList = {"_routing"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); @@ -828,7 +829,7 @@ void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolea " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"text1\": \"some text.\"\n," + @@ -890,7 +891,7 @@ void indexSingleDocumentWithNumericFieldsGeneratedFromText(boolean stored, boole " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"token_count\": \"A text with five words.\",\n" + diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java index 84a9fe2970bd0..7fa769b2613eb 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -40,10 +41,11 @@ public void testSetSource() throws InterruptedException, ExecutionException { map.put("test_field", "foobar"); IndexRequestBuilder[] builders = new IndexRequestBuilder[] { client().prepareIndex("test", "test").setSource((Object)"test_field", (Object)"foobar"), - client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))), + client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}", XContentType.JSON), + client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON), + client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON), + client().prepareIndex("test", "test") + .setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}")), XContentType.JSON), client().prepareIndex("test", "test").setSource(map) }; indexRandom(true, builders); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 28d67a4a1e2b6..80e453d665e3d 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.shard.IndexShard; @@ -232,7 +233,7 @@ public void testRefreshActuallyWorks() throws Exception { // now disable IndexMetaData metaData = IndexMetaData.builder(indexService.getMetaData()).settings(Settings.builder().put(indexService.getMetaData().getSettings()).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)).build(); indexService.updateMetaData(metaData); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 10); @@ -259,7 +260,7 @@ public void testAsyncFsyncActuallyWorks() throws Exception { IndexService indexService = createIndex("test", settings); ensureGreen("test"); assertTrue(indexService.getRefreshTask().mustReschedule()); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); assertBusy(() -> { assertFalse(shard.getTranslog().syncNeeded()); @@ -278,7 +279,7 @@ public void testRescheduleAsyncFsync() throws Exception { indexService.updateMetaData(metaData); assertNotNull(indexService.getFsyncTask()); assertTrue(indexService.getRefreshTask().mustReschedule()); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); assertBusy(() -> { assertFalse(shard.getTranslog().syncNeeded()); diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index ecdba6ec44f1e..c815b2b55f9f3 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexingSlowLog.SlowLogParsedDocumentPrinter; import org.elasticsearch.index.mapper.ParsedDocument; @@ -41,7 +42,7 @@ public class IndexingSlowLogTests extends ESTestCase { public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), SeqNoFieldMapper.SequenceID.emptySeqID(), "id", - "test", null, null, source, null); + "test", null, null, source, XContentType.JSON, null); Index index = new Index("foo", "123"); // Turning off document logging doesn't log source[] SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 0); diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java index 0f72e72f6a25f..ca089e6eb83ba 100644 --- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java +++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; @@ -95,14 +96,15 @@ public void testUpdate() throws InterruptedException, ExecutionException { assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); // Update with RefreshPolicy.WAIT_UNTIL - UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + UpdateResponse update = client().prepareUpdate("test", "test", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .get(); assertEquals(2, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); // Upsert with RefreshPolicy.WAIT_UNTIL - update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat") + update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat") .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); assertEquals(1, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); @@ -125,7 +127,7 @@ public void testBulk() { // Update by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz")); + bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz")); assertBulkSuccess(bulk.get()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 7c586b204b381..94b95d1aabf61 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -88,6 +88,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -290,7 +291,8 @@ private static ParsedDocument testParsedDocument(String id, String type, String document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingUpdate); } protected Store createStore() throws IOException { @@ -718,7 +720,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { public void testFlushIsDisabledDuringTranslogRecovery() throws IOException { assertFalse(engine.isRecovering()); - ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), SOURCE, null); engine.index(indexForDoc(doc)); engine.close(); @@ -727,7 +729,7 @@ public void testFlushIsDisabledDuringTranslogRecovery() throws IOException { assertTrue(engine.isRecovering()); engine.recoverFromTranslog(); assertFalse(engine.isRecovering()); - doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null); + doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), SOURCE, null); engine.index(indexForDoc(doc)); engine.flush(); } diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index 219af4af4daa9..53708b28dfb0f 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.Mapping; @@ -179,7 +180,8 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingsUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingsUpdate); } protected Store createStore(Path p) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java index 346b441470609..46749e792ed7d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.update.UpdateHelper; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -110,7 +111,7 @@ public static void destroyThreadPool() { public void testDynamicDisabled() { IndexRequest request = new IndexRequest("index", "type", "1"); - request.source("foo", 3); + request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(request); final AtomicBoolean onFailureCalled = new AtomicBoolean(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index f5c8d38503ebb..41256113cd6bf 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -197,7 +197,7 @@ private String serialize(ToXContent mapper) throws Exception { private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes()); + SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes(), builder.contentType()); try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) { ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser); assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); @@ -550,7 +550,7 @@ public void testMixTemplateMultiFieldAndMappingReuse() throws Exception { XContentBuilder json = XContentFactory.jsonBuilder().startObject() .field("field", "foo") .endObject(); - SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes()); + SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes(), json.contentType()); DocumentMapper mapper = indexService.mapperService().documentMapper("type1"); assertNull(mapper.mappers().getMapper("field.raw")); ParsedDocument parsed = mapper.parse(source); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 55b6e7bf71c30..b7ad6a7e4c3de 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; @@ -33,7 +34,7 @@ import static org.hamcrest.Matchers.nullValue; public class IdFieldMapperTests extends ESSingleNodeTestCase { - + public void testId() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); @@ -54,7 +55,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { try { docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder() - .startObject().field("_id", "1").endObject().bytes())); + .startObject().field("_id", "1").endObject().bytes(), XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 3c9a1c16d16b5..5543baff1a23b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -20,9 +20,11 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -91,7 +93,7 @@ public void testTypes() throws Exception { public void testIndexIntoDefaultMapping() throws Throwable { // 1. test implicit index creation ExecutionException e = expectThrows(ExecutionException.class, () -> { - client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}").execute().get(); + client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}", XContentType.JSON).execute().get(); }); Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause()); if (throwable instanceof IllegalArgumentException) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java index 5c70465d77f5c..8f17b3e0e0d64 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java @@ -186,7 +186,8 @@ public void testMultiFieldsInConsistentOrder() throws Exception { DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); Arrays.sort(multiFieldNames); - Map sourceAsMap = XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true).v2(); + Map sourceAsMap = + XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true, builder.contentType()).v2(); @SuppressWarnings("unchecked") Map multiFields = (Map) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap); assertThat(multiFields.size(), equalTo(multiFieldNames.length)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java index 225940d8eda3e..af5061e62ae95 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -53,7 +54,7 @@ public void testParentSetInDocNotAllowed() throws Exception { try { docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder() - .startObject().field("_parent", "1122").endObject().bytes())); + .startObject().field("_parent", "1122").endObject().bytes(), XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document")); @@ -72,13 +73,14 @@ public void testJoinFieldSet() throws Exception { // Indexing parent doc: DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type"); - ParsedDocument doc = parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"))); + ParsedDocument doc = + parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"), XContentType.JSON)); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); // Indexing child doc: DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type"); - doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122")); + doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}"), XContentType.JSON).parent("1122")); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); @@ -92,7 +94,7 @@ public void testJoinFieldNotSet() throws Exception { .startObject() .field("x_field", "x_value") .endObject() - .bytes())); + .bytes(), XContentType.JSON)); assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc())); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 92e93ede9aa7a..9c26a9806e331 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; @@ -40,7 +41,7 @@ public void testRoutingMapper() throws Exception { .startObject() .field("field", "value") .endObject() - .bytes()).routing("routing_value")); + .bytes(), XContentType.JSON).routing("routing_value")); assertThat(doc.rootDoc().get("_routing"), equalTo("routing_value")); assertThat(doc.rootDoc().get("field"), equalTo("value")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index adf05b9b33817..5803d7d957e1d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -144,7 +144,8 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception { .startObject("_source").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", new CompressedXContent(mapping), defaultMapping); + DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser() + .parse("my_type", new CompressedXContent(mapping), defaultMapping); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(true)); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java index ed0822d4d5391..67d7873b7b466 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java @@ -21,20 +21,16 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java index 8daf45ed0dd01..ee1265eca656f 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -26,17 +26,20 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; @@ -46,6 +49,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -301,6 +305,26 @@ public void testItemFromXContent() throws IOException { assertEquals(expectedItem, newItem); } + public void testItemSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEODXsiZm9vIjoiYmFyIn0A/wD//////////QAAAAAAAAAA"); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + Item item = new Item(in); + assertEquals(XContentType.JSON, item.xContentType()); + assertEquals("{\"foo\":\"bar\"}", item.doc().utf8ToString()); + assertEquals("index", item.index()); + assertEquals("type", item.type()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + item.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } + @Override protected boolean isCachable(MoreLikeThisQueryBuilder queryBuilder) { return queryBuilder.likeItems().length == 0; // items are always fetched diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index ede687d633bff..863c76bb7694b 100644 --- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.Uid; @@ -120,7 +121,7 @@ protected class ReplicationGroup implements AutoCloseable, Iterable public int indexDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet())) - .source("{}"); + .source("{}", XContentType.JSON); final IndexResponse response = index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); } @@ -130,7 +131,7 @@ public int indexDocs(final int numOfDoc) throws Exception { public int appendDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { - final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}"); + final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON); final IndexResponse response = index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); } diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index 41edc303f9dbd..39d8778c2a4b8 100644 --- a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.index.engine.InternalEngineTests; @@ -110,7 +111,7 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa public void testInheritMaxValidAutoIDTimestampOnRecovery() throws Exception { try (ReplicationGroup shards = createGroup(0)) { shards.startAll(); - final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}"); + final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON); indexRequest.onRetry(); // force an update of the timestamp final IndexResponse response = shards.index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 3e35ed357fff8..97c96c8af12f7 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; @@ -102,7 +103,8 @@ protected Collection> getPlugins() { } private ParsedDocument testParsedDocument(String id, String type, String routing, long seqNo, - ParseContext.Document document, BytesReference source, Mapping mappingUpdate) { + ParseContext.Document document, BytesReference source, XContentType xContentType, + Mapping mappingUpdate) { Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); @@ -112,7 +114,7 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); return new ParsedDocument(versionField, seqID, id, type, routing, - Collections.singletonList(document), source, mappingUpdate); + Collections.singletonList(document), source, xContentType, mappingUpdate); } public void testLockTryingToDelete() throws Exception { @@ -146,7 +148,7 @@ public void testLockTryingToDelete() throws Exception { public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0); @@ -163,14 +165,14 @@ public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { public void testDurableFlagHasEffect() { createIndex("test"); ensureGreen(); - client().prepareIndex("test", "bar", "1").setSource("{}").get(); + client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).get(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); setDurability(shard, Translog.Durability.REQUEST); assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.ASYNC); - client().prepareIndex("test", "bar", "2").setSource("{}").get(); + client().prepareIndex("test", "bar", "2").setSource("{}", XContentType.JSON).get(); assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); client().prepareDelete("test", "bar", "1").get(); @@ -181,13 +183,13 @@ public void testDurableFlagHasEffect() { assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "3").setSource("{}")) + .add(client().prepareIndex("test", "bar", "3").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "bar", "1")).get()); assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.ASYNC); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "4").setSource("{}")) + .add(client().prepareIndex("test", "bar", "4").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "bar", "3")).get()); setDurability(shard, Translog.Durability.REQUEST); assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); @@ -218,7 +220,7 @@ public void testIndexDirIsDeletedWhenShardRemoved() throws Exception { .build(); createIndex("test", idxSettings); ensureGreen("test"); - client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); SearchResponse response = client().prepareSearch("test").get(); assertHitCount(response, 1L); client().admin().indices().prepareDelete("test").get(); @@ -230,7 +232,7 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); for (int i = 0; i < 50; i++) { - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); } ensureGreen("test"); InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); @@ -266,7 +268,7 @@ public void testIndexCanChangeCustomDataPath() throws Exception { logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString()); createIndex(INDEX, sb); ensureGreen(INDEX); - client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex(INDEX, "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get(); assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L)); @@ -323,7 +325,8 @@ public void testMaybeFlush() throws Exception { client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(117 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); - client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "0") + .setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldFlush()); ParsedDocument doc = testParsedDocument( "1", @@ -331,12 +334,13 @@ public void testMaybeFlush() throws Exception { null, SequenceNumbersService.UNASSIGNED_SEQ_NO, new ParseContext.Document(), - new BytesArray(new byte[]{1}), null); + new BytesArray(new byte[]{1}), XContentType.JSON, null); Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); shard.index(index); assertTrue(shard.shouldFlush()); assertEquals(2, shard.getEngine().getTranslog().totalOperations()); - client().prepareIndex("test", "test", "2").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON) + .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertBusy(() -> { // this is async assertFalse(shard.shouldFlush()); }); @@ -369,7 +373,8 @@ public void testStressMaybeFlush() throws Exception { client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put( IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(117/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); - client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "0").setSource("{}", XContentType.JSON) + .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldFlush()); final AtomicBoolean running = new AtomicBoolean(true); final int numThreads = randomIntBetween(2, 4); @@ -394,7 +399,7 @@ public void run() { barrier.await(); FlushStats flushStats = shard.flushStats(); long total = flushStats.getTotal(); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); assertBusy(() -> assertEquals(total + 1, shard.flushStats().getTotal())); running.set(false); for (int i = 0; i < threads.length; i++) { @@ -409,9 +414,9 @@ public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexService(resolveIndex("test")); IndexShard shard = indexService.getShardOrNull(0); - client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get(); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); client().prepareDelete("test", "test", "0").get(); - client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {}; shard.close("simon says", false); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 79e3868da46e7..4a03a2632322d 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -558,7 +559,8 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingUpdate); } public void testIndexingOperationsListeners() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index d5c10dddc3653..846ca6be2014a 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; @@ -338,7 +339,8 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); BytesReference source = new BytesArray(new byte[] { 1 }); - ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, null); + ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, XContentType.JSON, + null); Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); return engine.index(index); } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 1a9d7c97dc355..0591500423902 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; @@ -2028,7 +2029,8 @@ public void testTranslogOpSerialization() throws Exception { document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, null); + ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, XContentType.JSON, + null); Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm, 1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false); diff --git a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index 82482b1f821f0..fe4ab2e363a88 100644 --- a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -57,9 +58,9 @@ public void testIndexNameDateMathExpressions() { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get(); - client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get(); - client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get(); + client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); @@ -116,9 +117,9 @@ public void testAutoCreateIndexWithDateMathExpression() throws Exception { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get(); - client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get(); - client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get(); + client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index e9a717f663676..fee2964fe126e 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexSearcherWrapper; @@ -423,7 +424,7 @@ public void testTranslogRecoveryWorksWithIMC() throws IOException { IndexService indexService = indicesService.indexService(resolveIndex("test")); IndexShard shard = indexService.getShardOrNull(0); for (int i = 0; i < 100; i++) { - client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}").get(); + client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); } IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {}; diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 012fae17f5c8d..61d325a64e8f7 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.Analysis; @@ -319,7 +320,8 @@ public void testWordListPath() throws Exception { String[] words = new String[]{"donau", "dampf", "schiff", "spargel", "creme", "suppe"}; Path wordListFile = generateWordList(words); - settings = Settings.builder().loadFromSource("index: \n word_list_path: " + wordListFile.toAbsolutePath()).build(); + settings = Settings.builder().loadFromSource("index: \n word_list_path: " + wordListFile.toAbsolutePath(), XContentType.YAML) + .build(); Set wordList = Analysis.getWordSet(env, Version.CURRENT, settings, "index.word_list"); MatcherAssert.assertThat(wordList.size(), equalTo(6)); diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index d974ea348cfac..50979cf611d07 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; @@ -57,7 +58,7 @@ public void testWaitIfOngoing() throws InterruptedException { final int numIters = scaledRandomIntBetween(10, 30); for (int i = 0; i < numIters; i++) { for (int j = 0; j < 10; j++) { - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); } final CountDownLatch latch = new CountDownLatch(10); final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); @@ -164,7 +165,7 @@ public void testSyncedFlushWithConcurrentIndexing() throws Exception { @Override public void run() { while (stop.get() == false) { - client().prepareIndex().setIndex("test").setType("doc").setSource("{}").get(); + client().prepareIndex().setIndex("test").setType("doc").setSource("{}", XContentType.JSON).get(); numDocs.incrementAndGet(); } } diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java index c35df81cfd673..59784456e988b 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; @@ -42,7 +43,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { public void testModificationPreventsFlushing() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -54,7 +55,7 @@ public void testModificationPreventsFlushing() throws InterruptedException { assertEquals("exactly one active shard", 1, activeShards.size()); Map commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId); assertEquals("exactly one commit id", 1, commitIds.size()); - client().prepareIndex("test", "test", "2").setSource("{}").get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get(); String syncId = UUIDs.base64UUID(); SyncedFlushUtil.LatchedListener listener = new SyncedFlushUtil.LatchedListener<>(); flushService.sendSyncRequests(syncId, activeShards, state, commitIds, shardId, shardRoutingTable.size(), listener); @@ -86,7 +87,7 @@ public void testModificationPreventsFlushing() throws InterruptedException { public void testSingleShardSuccess() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -106,7 +107,7 @@ public void testSingleShardSuccess() throws InterruptedException { public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -161,7 +162,7 @@ public void testSyncFailsOnIndexClosedOrMissing() throws InterruptedException { public void testFailAfterIntermediateCommit() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -174,7 +175,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException { Map commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId); assertEquals("exactly one commit id", 1, commitIds.size()); if (randomBoolean()) { - client().prepareIndex("test", "test", "2").setSource("{}").get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get(); } client().admin().indices().prepareFlush("test").setForce(true).get(); String syncId = UUIDs.base64UUID(); @@ -194,7 +195,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException { public void testFailWhenCommitIsMissing() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 0ef915b8276fc..484f6e5db76aa 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -40,6 +41,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.CircuitBreakerStats; @@ -177,7 +179,7 @@ public void testRamAccountingTermsEnum() throws Exception { // Create an index where the mappings have a field data filter assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " + - "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}")); + "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", XContentType.JSON)); ensureGreen("ramtest"); @@ -480,7 +482,7 @@ public void testLimitsRequestSize() throws Exception { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)); - indexRequest.source("field", "value", "num", i); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i); bulkRequest.add(indexRequest); } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 424edb42e68d1..72a789ddc1c1d 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.store.Store; @@ -576,7 +577,7 @@ public void testDisconnectsWhileRecovering() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); ensureSearchable(indexName); @@ -686,7 +687,7 @@ public void testDisconnectsDuringRecovery() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); ensureSearchable(indexName); diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java index 1d266698db1dc..bbe4d8ed12e80 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.SourceToParse; @@ -43,7 +44,7 @@ public void testGetStartingSeqNo() throws Exception { long seqNo = 0; for (int i = 0; i < docs; i++) { Engine.Index indexOp = replica.prepareIndexOnReplica( - SourceToParse.source(SourceToParse.Origin.REPLICA, index, "type", "doc_" + i, new BytesArray("{}")), + SourceToParse.source(SourceToParse.Origin.REPLICA, index, "type", "doc_" + i, new BytesArray("{}"), XContentType.JSON), seqNo++, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); replica.index(indexOp); if (rarely()) { diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 519df00c06c69..91673630c3d49 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; @@ -739,8 +740,8 @@ public void testFieldDataFieldsParam() throws Exception { ensureGreen(); - client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); - client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); refresh(); client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet(); @@ -786,8 +787,8 @@ public void testCompletionFieldsParam() throws Exception { "{ \"properties\": { \"bar\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}")); ensureGreen(); - client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); - client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); refresh(); IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); @@ -1125,7 +1126,7 @@ public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierExcepti final IndexResponse response = client() .prepareIndex("test", "type", id) - .setSource("{}") + .setSource("{}", XContentType.JSON) .get(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 6469b660f0b98..cb6b8b752bcf3 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -30,8 +30,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.InvalidAliasNameException; @@ -335,7 +337,7 @@ public void testBrokenMapping() throws Exception { MapperParsingException e = expectThrows( MapperParsingException.class, () -> client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "abcde") + .addMapping("type1", "{\"foo\": \"abcde\"}", XContentType.JSON) .get()); assertThat(e.getMessage(), containsString("Failed to parse mapping ")); @@ -372,7 +374,7 @@ public void testIndexTemplateWithAliases() throws Exception { client().admin().indices().preparePutTemplate("template_with_aliases") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}") + .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON) .addAlias(new Alias("simple_alias")) .addAlias(new Alias("templated_alias-{index}")) .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}")) @@ -423,7 +425,7 @@ public void testIndexTemplateWithAliases() throws Exception { public void testIndexTemplateWithAliasesInSource() { client().admin().indices().preparePutTemplate("template_1") - .setSource("{\n" + + .setSource(new BytesArray("{\n" + " \"template\" : \"*\",\n" + " \"aliases\" : {\n" + " \"my_alias\" : {\n" + @@ -434,7 +436,7 @@ public void testIndexTemplateWithAliasesInSource() { " }\n" + " }\n" + " }\n" + - "}").get(); + "}"), XContentType.JSON).get(); assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2")); @@ -635,8 +637,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio .setOrder(0) .addAlias(new Alias("alias4").filter(termQuery("field", "value"))).get(); - client().prepareIndex("a1", "test", "test").setSource("{}").get(); - BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get(); + client().prepareIndex("a1", "test", "test").setSource("{}", XContentType.JSON).get(); + BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("a2")); @@ -644,8 +646,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio assertThat(response.getItems()[0].getId(), equalTo("test")); assertThat(response.getItems()[0].getVersion(), equalTo(1L)); - client().prepareIndex("b1", "test", "test").setSource("{}").get(); - response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get(); + client().prepareIndex("b1", "test", "test").setSource("{}", XContentType.JSON).get(); + response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("b2")); @@ -653,8 +655,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio assertThat(response.getItems()[0].getId(), equalTo("test")); assertThat(response.getItems()[0].getVersion(), equalTo(1L)); - client().prepareIndex("c1", "test", "test").setSource("{}").get(); - response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get(); + client().prepareIndex("c1", "test", "test").setSource("{}", XContentType.JSON).get(); + response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("c2")); @@ -670,9 +672,9 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio // So the aliases defined in the index template for this index will not fail // even though the fields in the alias fields don't exist yet and indexing into // an index that doesn't exist yet will succeed - client().prepareIndex("d1", "test", "test").setSource("{}").get(); + client().prepareIndex("d1", "test", "test").setSource("{}", XContentType.JSON).get(); - response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get(); + response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getId(), equalTo("test")); @@ -702,7 +704,7 @@ public void testCombineTemplates() throws Exception{ " }\n" + " }\n" + " }\n" + - " }\n") + " }\n", XContentType.JSON) .get(); // put template using custom_1 analyzer diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 14335bde076de..2b59f0d421cb9 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -36,8 +36,10 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -81,7 +83,7 @@ public void testSimulate() throws Exception { .endObject() .endArray() .endObject().bytes(); - client().admin().cluster().preparePutPipeline("_id", pipelineSource) + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON) .get(); GetPipelineResponse getResponse = client().admin().cluster().prepareGetPipeline("_id") .get(); @@ -104,10 +106,10 @@ public void testSimulate() throws Exception { .endObject().bytes(); SimulatePipelineResponse response; if (randomBoolean()) { - response = client().admin().cluster().prepareSimulatePipeline(bytes) + response = client().admin().cluster().prepareSimulatePipeline(bytes, XContentType.JSON) .setId("_id").get(); } else { - SimulatePipelineRequest request = new SimulatePipelineRequest(bytes); + SimulatePipelineRequest request = new SimulatePipelineRequest(bytes, XContentType.JSON); request.setId("_id"); response = client().admin().cluster().simulatePipeline(request).get(); } @@ -137,14 +139,14 @@ public void testBulkWithIngestFailures() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); client().admin().cluster().putPipeline(putPipelineRequest).get(); int numRequests = scaledRandomIntBetween(32, 128); BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id"); - indexRequest.source("field", "value", "fail", i % 2 == 0); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0); bulkRequest.add(indexRequest); } @@ -177,7 +179,7 @@ public void test() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); client().admin().cluster().putPipeline(putPipelineRequest).get(); GetPipelineRequest getPipelineRequest = new GetPipelineRequest("_id"); @@ -219,7 +221,7 @@ public void testPutWithPipelineFactoryError() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); try { client().admin().cluster().putPipeline(putPipelineRequest).get(); } catch (ExecutionException e) { diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java index e2fca8b4112a9..518b775d7f802 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java @@ -42,10 +42,10 @@ public class IngestMetadataTests extends ESTestCase { public void testFromXContent() throws IOException { PipelineConfiguration pipeline = new PipelineConfiguration( - "1",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "1", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); PipelineConfiguration pipeline2 = new PipelineConfiguration( - "2",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}") + "2", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}"), XContentType.JSON ); Map map = new HashMap<>(); map.put(pipeline.getId(), pipeline); @@ -72,14 +72,14 @@ public void testDiff() throws Exception { BytesReference pipelineConfig = new BytesArray("{}"); Map pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", pipelineConfig)); - pipelines.put("2", new PipelineConfiguration("2", pipelineConfig)); + pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", pipelineConfig, XContentType.JSON)); IngestMetadata ingestMetadata1 = new IngestMetadata(pipelines); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", pipelineConfig)); - pipelines.put("3", new PipelineConfiguration("3", pipelineConfig)); - pipelines.put("4", new PipelineConfiguration("4", pipelineConfig)); + pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)); + pipelines.put("3", new PipelineConfiguration("3", pipelineConfig, XContentType.JSON)); + pipelines.put("4", new PipelineConfiguration("4", pipelineConfig, XContentType.JSON)); IngestMetadata ingestMetadata2 = new IngestMetadata(pipelines); IngestMetadata.IngestMetadataDiff diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata2.diff(ingestMetadata1); @@ -92,13 +92,13 @@ public void testDiff() throws Exception { IngestMetadata endResult = (IngestMetadata) diff.apply(ingestMetadata2); assertThat(endResult, not(equalTo(ingestMetadata1))); assertThat(endResult.getPipelines().size(), equalTo(3)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig))); - assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig, XContentType.JSON))); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"))); - pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"))); + pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"), XContentType.JSON)); IngestMetadata ingestMetadata3 = new IngestMetadata(pipelines); diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata3.diff(ingestMetadata1); @@ -108,12 +108,12 @@ public void testDiff() throws Exception { endResult = (IngestMetadata) diff.apply(ingestMetadata3); assertThat(endResult, equalTo(ingestMetadata1)); assertThat(endResult.getPipelines().size(), equalTo(2)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig, XContentType.JSON))); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"))); - pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"))); + pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON)); IngestMetadata ingestMetadata4 = new IngestMetadata(pipelines); diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata4.diff(ingestMetadata1); @@ -123,7 +123,8 @@ public void testDiff() throws Exception { endResult = (IngestMetadata) diff.apply(ingestMetadata4); assertThat(endResult, not(equalTo(ingestMetadata1))); assertThat(endResult.getPipelines().size(), equalTo(2)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}")))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("2"), + equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON))); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index 83072636b762b..1a8e6e873e01d 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.node.NodeService; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -69,7 +70,7 @@ public void testFailPipelineCreation() throws Exception { ensureStableCluster(2, node2); try { - client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); fail("exception expected"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("Processor type [test] is not installed on node")); @@ -82,7 +83,7 @@ public void testFailPipelineCreationProcessorNotInstalledOnMasterNode() throws E internalCluster().startNode(); try { - client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("No processor type exists with name [test]")); @@ -95,7 +96,7 @@ public void testFailStartNode() throws Exception { installPlugin = true; String node1 = internalCluster().startNode(); - WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); assertThat(response.isAcknowledged(), is(true)); Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id"); assertThat(pipeline, notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java new file mode 100644 index 0000000000000..f41d01b32c89d --- /dev/null +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class PipelineConfigurationTests extends ESTestCase { + + public void testSerialization() throws IOException { + PipelineConfiguration configuration = new PipelineConfiguration("1", + new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, configuration.getXContentType()); + + BytesStreamOutput out = new BytesStreamOutput(); + configuration.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + PipelineConfiguration serialized = PipelineConfiguration.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getConfig().utf8ToString()); + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ATECe30AAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PipelineConfiguration configuration = PipelineConfiguration.readFrom(in); + assertEquals(XContentType.JSON, configuration.getXContentType()); + assertEquals("{}", configuration.getConfig().utf8ToString()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + configuration.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } + + public void testParser() throws IOException { + ContextParser parser = PipelineConfiguration.getParser(); + XContentType xContentType = randomFrom(XContentType.values()); + final BytesReference bytes; + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + new PipelineConfiguration("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON) + .toXContent(builder, ToXContent.EMPTY_PARAMS); + bytes = builder.bytes(); + } + + XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes); + PipelineConfiguration parsed = parser.parse(xContentParser, null); + assertEquals(xContentType, parsed.getXContentType()); + assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); + assertEquals("1", parsed.getId()); + } +} diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java index c00e9254ab289..7ad445e301370 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java @@ -25,8 +25,10 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.CustomTypeSafeMatcher; @@ -273,7 +275,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception { } } else { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId); - indexRequest.source("field1", "value1"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); request = indexRequest; numIndexRequests++; } @@ -303,7 +305,7 @@ public void testBulkRequestExecution() throws Exception { int numRequest = scaledRandomIntBetween(8, 64); for (int i = 0; i < numRequest; i++) { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId); - indexRequest.source("field1", "value1"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); bulkRequest.add(indexRequest); } @@ -331,8 +333,8 @@ public void testStats() throws Exception { when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, null, new CompoundProcessor(mock(Processor.class)))); Map configurationMap = new HashMap<>(); - configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); - configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"))); + configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON)); + configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); @SuppressWarnings("unchecked") @@ -361,14 +363,14 @@ public void testStats() throws Exception { // issue: https://github.com/elastic/elasticsearch/issues/18126 public void testUpdatingStatsWhenRemovingPipelineWorks() throws Exception { Map configurationMap = new HashMap<>(); - configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); - configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"))); + configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON)); + configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id1")); assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id2")); configurationMap = new HashMap<>(); - configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"))); + configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id1"))); assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id2"))); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java index 5967354710c7b..e91bb7e2e6e17 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -102,7 +103,7 @@ public void testUpdatePipelines() { assertThat(store.pipelines.size(), is(0)); PipelineConfiguration pipeline = new PipelineConfiguration( - "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", pipeline)); clusterState = ClusterState.builder(clusterState) @@ -123,7 +124,7 @@ public void testPut() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // add a new pipeline: - PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}")); + PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -134,7 +135,8 @@ public void testPut() { assertThat(pipeline.getProcessors().size(), equalTo(0)); // overwrite existing pipeline: - putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}")); + putRequest = + new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"), XContentType.JSON); previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -151,7 +153,8 @@ public void testPutWithErrorResponse() { assertThat(pipeline, nullValue()); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); - PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}")); + PutPipelineRequest putRequest = + new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); try { @@ -166,7 +169,7 @@ public void testPutWithErrorResponse() { public void testDelete() { PipelineConfiguration config = new PipelineConfiguration( - "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config)); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); @@ -197,9 +200,9 @@ public void testDeleteUsingWildcard() { BytesArray definition = new BytesArray( "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}" ); - pipelines.put("p1", new PipelineConfiguration("p1", definition)); - pipelines.put("p2", new PipelineConfiguration("p2", definition)); - pipelines.put("q1", new PipelineConfiguration("q1", definition)); + pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON)); + pipelines.put("p2", new PipelineConfiguration("p2", definition, XContentType.JSON)); + pipelines.put("q1", new PipelineConfiguration("q1", definition, XContentType.JSON)); IngestMetadata ingestMetadata = new IngestMetadata(pipelines); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); ClusterState previousClusterState = clusterState; @@ -245,7 +248,7 @@ public void testDeleteWithExistingUnmatchedPipelines() { BytesArray definition = new BytesArray( "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}" ); - pipelines.put("p1", new PipelineConfiguration("p1", definition)); + pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON)); IngestMetadata ingestMetadata = new IngestMetadata(pipelines); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); ClusterState previousClusterState = clusterState; @@ -266,10 +269,10 @@ public void testDeleteWithExistingUnmatchedPipelines() { public void testGetPipelines() { Map configs = new HashMap<>(); configs.put("_id1", new PipelineConfiguration( - "_id1", new BytesArray("{\"processors\": []}") + "_id1", new BytesArray("{\"processors\": []}"), XContentType.JSON )); configs.put("_id2", new PipelineConfiguration( - "_id2", new BytesArray("{\"processors\": []}") + "_id2", new BytesArray("{\"processors\": []}"), XContentType.JSON )); assertThat(store.innerGetPipelines(null, "_id1").isEmpty(), is(true)); @@ -311,7 +314,7 @@ public void testCrud() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty PutPipelineRequest putRequest = new PutPipelineRequest(id, - new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")); + new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -332,7 +335,8 @@ public void testCrud() throws Exception { public void testValidate() throws Exception { PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray( - "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}")); + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"), + XContentType.JSON); DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -355,7 +359,7 @@ public void testValidate() throws Exception { public void testValidateNoIngestInfo() throws Exception { PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray( - "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")); + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON); try { store.validatePipeline(Collections.emptyMap(), putRequest); fail("exception expected"); diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java index 5c499eae4d8a9..3f0eac7762f0b 100644 --- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java +++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.test.ESIntegTestCase; @@ -143,7 +144,7 @@ public void testThatSourceFilteringIsSupported() throws Exception { .field("excluded", "should not be seen") .endObject().bytes(); for (int i = 0; i < 100; i++) { - client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef).get(); + client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef, XContentType.JSON).get(); } MultiGetRequestBuilder request = client().prepareMultiGet(); diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 8d19f909580c2..56f9b265c0dfc 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbersService; @@ -334,12 +335,12 @@ public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardSt List builders1 = new ArrayList<>(); for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) { - builders1.add(client().prepareIndex("test", "type").setSource("{}")); + builders1.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON)); } List builders2 = new ArrayList<>(); for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) { - builders2.add(client().prepareIndex("test", "type").setSource("{}")); + builders2.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON)); } logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]); @@ -392,7 +393,7 @@ public void testCancellationCleansTempFiles() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").setWaitForGreenStatus().get().isTimedOut()); diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java index e860cfc9ebd54..bf8bce8ae6334 100644 --- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.client.Requests.flushRequest; @@ -52,12 +53,12 @@ public void testSimpleRecovery() throws Exception { NumShards numShards = getNumShards("test"); - client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet(); + client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); FlushResponse flushResponse = client().admin().indices().flush(flushRequest("test")).actionGet(); assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(flushResponse.getFailedShards(), equalTo(0)); - client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"))).actionGet(); + client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"), XContentType.JSON)).actionGet(); RefreshResponse refreshResponse = client().admin().indices().refresh(refreshRequest("test")).actionGet(); assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 4e8d9b8600f72..431bc152bd9eb 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -165,7 +165,7 @@ public void testConvert() throws IOException { public void testResponseWhenPathContainsEncodingError() throws IOException { final String path = "%a"; - final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path) { + final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, Collections.emptyMap()) { @Override public Method method() { return null; @@ -185,16 +185,6 @@ public boolean hasContent() { public BytesReference content() { return null; } - - @Override - public String header(String name) { - return null; - } - - @Override - public Iterable> headers() { - return null; - } }; final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath())); final RestChannel channel = new DetailedExceptionRestChannel(request); diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index e7064554908f3..79c6f6561e6fa 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,10 +42,11 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.http.HttpInfo; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.HttpStats; -import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -91,19 +93,17 @@ public void testApplyRelevantHeaders() throws Exception { final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); Set headers = new HashSet<>(Arrays.asList("header.1", "header.2")); final RestController restController = new RestController(Settings.EMPTY, headers, null, null, circuitBreakerService); - restController.registerHandler(RestRequest.Method.GET, "/", - (RestRequest request, RestChannel channel, NodeClient client) -> { + threadContext.putHeader("header.3", "true"); + Map> restHeaders = new HashMap<>(); + restHeaders.put("header.1", Collections.singletonList("true")); + restHeaders.put("header.2", Collections.singletonList("true")); + restHeaders.put("header.3", Collections.singletonList("false")); + restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null, + threadContext, (RestRequest request, RestChannel channel, NodeClient client) -> { assertEquals("true", threadContext.getHeader("header.1")); assertEquals("true", threadContext.getHeader("header.2")); assertNull(threadContext.getHeader("header.3")); }); - threadContext.putHeader("header.3", "true"); - Map restHeaders = new HashMap<>(); - restHeaders.put("header.1", "true"); - restHeaders.put("header.2", "true"); - restHeaders.put("header.3", "false"); - restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null, - threadContext); assertNull(threadContext.getHeader("header.1")); assertNull(threadContext.getHeader("header.2")); assertEquals("true", threadContext.getHeader("header.3")); @@ -172,9 +172,8 @@ public void testRestHandlerWrapper() throws Exception { }; final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper, null, circuitBreakerService); - restController.registerHandler(RestRequest.Method.GET, "/", handler); final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext); + restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext, handler); assertTrue(wrapperCalled.get()); assertFalse(handlerCalled.get()); } @@ -203,7 +202,7 @@ public boolean canTripCircuitBreaker() { public void testDispatchRequestAddsAndFreesBytesOnSuccess() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/", content); + TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -215,7 +214,7 @@ public void testDispatchRequestAddsAndFreesBytesOnSuccess() { public void testDispatchRequestAddsAndFreesBytesOnError() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/error", content); + TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -228,7 +227,7 @@ public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); // we will produce an error in the rest handler and one more when sending the error response - TestRestRequest request = new TestRestRequest("/error", content); + TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON); ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -240,7 +239,7 @@ public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() { public void testDispatchRequestLimitsBytes() { int contentLength = BREAKER_LIMIT.bytesAsInt() + 1; String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/", content); + TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -249,6 +248,67 @@ public void testDispatchRequestLimitsBytes() { assertEquals(0, inFlightRequestsBreaker.getUsed()); } + public void testDispatchRequiresContentTypeForRequestsWithContent() { + String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); + TestRestRequest request = new TestRestRequest("/", content, null); + AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); + restController = new RestController( + Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(), + Collections.emptySet(), null, null, circuitBreakerService); + restController.registerHandler(RestRequest.Method.GET, "/", + (r, c, client) -> c.sendResponse( + new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY))); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + } + + public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() { + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + } + + public void testDispatchWorksWithPlainText() { + String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withContent(new BytesArray(content), null).withPath("/foo") + .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + } + + @Override + public boolean supportsPlainText() { + return true; + } + }); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + assertWarnings("Plain text request bodies are deprecated. Use request parameters or body in a supported format."); + } + + public void testDispatchWorksWithAutoDetection() { + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withContent(new BytesArray("{}"), null).withPath("/") + .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("application/x-www-form-urlencoded"))).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header."); + } + private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { @@ -287,6 +347,7 @@ public HttpStats stats() { private static final class AssertingChannel extends AbstractRestChannel { private final RestStatus expectedStatus; + private final AtomicBoolean sendResponseCalled = new AtomicBoolean(false); protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) { super(request, detailedErrorsEnabled); @@ -296,6 +357,7 @@ protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, R @Override public void sendResponse(RestResponse response) { assertEquals(expectedStatus, response.status()); + sendResponseCalled.set(true); } } @@ -315,8 +377,9 @@ private static final class TestRestRequest extends RestRequest { private final BytesReference content; - private TestRestRequest(String path, String content) { - super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path); + private TestRestRequest(String path, String content, XContentType xContentType) { + super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, xContentType == null ? + Collections.emptyMap() : Collections.singletonMap("Content-Type", Collections.singletonList(xContentType.mediaType()))); this.content = new BytesArray(content); } @@ -340,15 +403,5 @@ public BytesReference content() { return content; } - @Override - public String header(String name) { - return null; - } - - @Override - public Iterable> headers() { - return null; - } - } } diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java index 7f0184e20c60b..bddcd39160a89 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -24,11 +24,15 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; @@ -54,10 +58,13 @@ public void testApplyContentParser() throws IOException { } public void testContentOrSourceParam() throws IOException { - assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("", singletonMap("source", "stuff")).contentOrSourceParam()); + assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam().v2()); + assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam().v2()); + assertEquals(new BytesArray("stuff"), + new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam().v2()); + assertEquals(new BytesArray("{\"foo\": \"stuff\"}"), + new ContentRestRequest("", singletonMap("source", "{\"foo\": \"stuff\"}")).contentOrSourceParam().v2()); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); } public void testHasContentOrSourceParam() throws IOException { @@ -74,6 +81,7 @@ public void testContentOrSourceParamParser() throws IOException { assertEquals(emptyMap(), new ContentRestRequest("{}", emptyMap()).contentOrSourceParamParser().map()); assertEquals(emptyMap(), new ContentRestRequest("{}", singletonMap("source", "stuff2")).contentOrSourceParamParser().map()); assertEquals(emptyMap(), new ContentRestRequest("", singletonMap("source", "{}")).contentOrSourceParamParser().map()); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); } public void testWithContentOrSourceParamParserOrNull() throws IOException { @@ -83,12 +91,58 @@ public void testWithContentOrSourceParamParserOrNull() throws IOException { assertEquals(emptyMap(), parser.map())); new ContentRestRequest("", singletonMap("source", "{}")).withContentOrSourceParamParserOrNull(parser -> assertEquals(emptyMap(), parser.map())); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); + } + + public void testContentTypeParsing() { + for (XContentType xContentType : XContentType.values()) { + Map> map = new HashMap<>(); + map.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + ContentRestRequest restRequest = new ContentRestRequest("", Collections.emptyMap(), map); + assertEquals(xContentType, restRequest.getXContentType()); + + map = new HashMap<>(); + map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters())); + restRequest = new ContentRestRequest("", Collections.emptyMap(), map); + assertEquals(xContentType, restRequest.getXContentType()); + } + } + + public void testPlainTextSupport() { + ContentRestRequest restRequest = new ContentRestRequest(randomAsciiOfLengthBetween(1, 30), Collections.emptyMap(), + Collections.singletonMap("Content-Type", + Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8")))); + assertNull(restRequest.getXContentType()); + } + + public void testMalformedContentTypeHeader() { + final String type = randomFrom("text", "text/:ain; charset=utf-8", "text/plain\";charset=utf-8", ":", "/", "t:/plain"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(), + Collections.singletonMap("Content-Type", Collections.singletonList(type)))); + assertEquals("invalid Content-Type header [" + type + "]", e.getMessage()); + } + + public void testNoContentTypeHeader() { + ContentRestRequest contentRestRequest = new ContentRestRequest("", Collections.emptyMap(), Collections.emptyMap()); + assertNull(contentRestRequest.getXContentType()); + } + + public void testMultipleContentTypeHeaders() { + List headers = new ArrayList<>(randomUnique(() -> randomAsciiOfLengthBetween(1, 16), randomIntBetween(2, 10))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(), + Collections.singletonMap("Content-Type", headers))); + assertEquals("only one Content-Type header should be provided", e.getMessage()); } private static final class ContentRestRequest extends RestRequest { private final BytesArray content; - public ContentRestRequest(String content, Map params) { - super(NamedXContentRegistry.EMPTY, params, "not used by this test"); + + ContentRestRequest(String content, Map params) { + this(content, params, Collections.singletonMap("Content-Type", Collections.singletonList("application/json"))); + } + + ContentRestRequest(String content, Map params, Map> headers) { + super(NamedXContentRegistry.EMPTY, params, "not used by this test", headers); this.content = new BytesArray(content); } @@ -96,7 +150,7 @@ public ContentRestRequest(String content, Map params) { public boolean hasContent() { return Strings.hasLength(content); } - + @Override public BytesReference content() { return content; @@ -111,15 +165,5 @@ public String uri() { public Method method() { throw new UnsupportedOperationException("Not used by this test"); } - - @Override - public Iterable> headers() { - throw new UnsupportedOperationException("Not used by this test"); - } - - @Override - public String header(String name) { - throw new UnsupportedOperationException("Not used by this test"); - } } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java index da52358d3c3c2..958b9e5222fad 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; @@ -92,7 +93,8 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception { RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java index 38ed76cae5249..4f7bc1bedd9d6 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java @@ -104,35 +104,35 @@ public void testThatDisplayHeadersAreNotAddedTwice() throws Exception { } public void testThatWeUseTheAcceptHeaderJson() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_JSON), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY); } public void testThatWeUseTheAcceptHeaderYaml() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_YAML), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY); } public void testThatWeUseTheAcceptHeaderSmile() throws Exception { - assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_SMILE), + assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE); } public void testThatWeUseTheAcceptHeaderCbor() throws Exception { - assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_CBOR), + assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_CBOR)), APPLICATION_CBOR); } public void testThatWeUseTheAcceptHeaderText() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, TEXT_PLAIN), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY); } public void testIgnoreContentType() throws Exception { - assertResponse(Collections.singletonMap(CONTENT_TYPE, APPLICATION_JSON), + assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY); } @@ -252,7 +252,7 @@ public void testMultiSort() { assertEquals(Arrays.asList(1,0,2), rowOrder); } - private RestResponse assertResponseContentType(Map headers, String mediaType) throws Exception { + private RestResponse assertResponseContentType(Map> headers, String mediaType) throws Exception { FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build(); table.startRow(); table.addCell("foo"); @@ -274,7 +274,7 @@ public void sendResponse(RestResponse response) { return response; } - private void assertResponse(Map headers, String mediaType, String body) throws Exception { + private void assertResponse(Map> headers, String mediaType, String body) throws Exception { RestResponse response = assertResponseContentType(headers, mediaType); assertThat(response.content().utf8ToString(), equalTo(body)); } diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index 621ade9e31de9..dc8b6b54464e4 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -64,7 +65,7 @@ public void testAliasCrudRouting() throws Exception { logger.info("--> updating with id [1] and routing through alias"); client().prepareUpdate("alias0", "type1", "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setDoc("field", "value2") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") .execute().actionGet(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("alias0", "type1", "1").execute().actionGet().isExists(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 2490134db4e85..b027a13e867e1 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -207,13 +207,13 @@ public void testRequiredRoutingCrudApis() throws Exception { } try { - client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field", "value2").execute().actionGet(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").execute().actionGet(); fail("update with missing routing when routing is required should fail"); } catch(ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); } - client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc("field", "value2").execute().actionGet(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); client().admin().indices().prepareRefresh().execute().actionGet(); for (int i = 0; i < 5; i++) { @@ -254,7 +254,7 @@ public void testRequiredRoutingBulk() throws Exception { { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1") - .source("field", "value")).execute().actionGet(); + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -269,12 +269,13 @@ public void testRequiredRoutingBulk() throws Exception { { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").routing("0") - .source("field", "value")).execute().actionGet(); + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } { - BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2")) + BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1") + .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2")) .execute().actionGet(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -289,7 +290,8 @@ public void testRequiredRoutingBulk() throws Exception { } { - BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2") + BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1") + .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2") .routing("0")).execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } @@ -369,12 +371,12 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { } UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0") - .setDoc("field1", "value1").get(); + .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.getVersion(), equalTo(2L)); try { - client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field1", "value1").get(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); fail(); } catch (RoutingMissingException e) { assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java index 5365d75bdf968..1dd277f98d72f 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java @@ -38,23 +38,23 @@ public void testGetScript() throws Exception { XContentBuilder sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().startObject("template").field("field", "value").endObject().endObject(); - builder.storeScript("template", StoredScriptSource.parse("lang", sourceBuilder.bytes())); + builder.storeScript("template", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType())); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("template", "value").endObject(); - builder.storeScript("template_field", StoredScriptSource.parse("lang", sourceBuilder.bytes())); + builder.storeScript("template_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType())); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().startObject("script").field("field", "value").endObject().endObject(); - builder.storeScript("script", StoredScriptSource.parse("lang", sourceBuilder.bytes())); + builder.storeScript("script", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType())); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("script", "value").endObject(); - builder.storeScript("script_field", StoredScriptSource.parse("lang", sourceBuilder.bytes())); + builder.storeScript("script_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType())); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("field", "value").endObject(); - builder.storeScript("any", StoredScriptSource.parse("lang", sourceBuilder.bytes())); + builder.storeScript("any", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType())); ScriptMetaData scriptMetaData = builder.build(); assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("template", "lang").getCode()); @@ -66,15 +66,15 @@ public void testGetScript() throws Exception { public void testDiff() throws Exception { ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null); - builder.storeScript("1", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"abc\"}"))); - builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"def\"}"))); - builder.storeScript("3", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"ghi\"}"))); + builder.storeScript("1", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"abc\"}"), XContentType.JSON)); + builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"def\"}"), XContentType.JSON)); + builder.storeScript("3", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"ghi\"}"), XContentType.JSON)); ScriptMetaData scriptMetaData1 = builder.build(); builder = new ScriptMetaData.Builder(scriptMetaData1); - builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"changed\"}"))); + builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"changed\"}"), XContentType.JSON)); builder.deleteScript("3", "lang"); - builder.storeScript("4", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"jkl\"}"))); + builder.storeScript("4", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"jkl\"}"), XContentType.JSON)); ScriptMetaData scriptMetaData2 = builder.build(); ScriptMetaData.ScriptMetadataDiff diff = (ScriptMetaData.ScriptMetadataDiff) scriptMetaData2.diff(scriptMetaData1); @@ -93,7 +93,7 @@ public void testDiff() throws Exception { public void testBuilder() { ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null); - builder.storeScript("_id", StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"1 + 1\"}"))); + builder.storeScript("_id", StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"1 + 1\"}"), XContentType.JSON)); ScriptMetaData result = builder.build(); assertEquals("1 + 1", result.getStoredScript("_id", "_lang").getCode()); @@ -106,7 +106,8 @@ private ScriptMetaData randomScriptMetaData(XContentType sourceContentType) thro String lang = randomAsciiOfLength(4); XContentBuilder sourceBuilder = XContentBuilder.builder(sourceContentType.xContent()); sourceBuilder.startObject().field("script", randomAsciiOfLength(4)).endObject(); - builder.storeScript(randomAsciiOfLength(i + 1), StoredScriptSource.parse(lang, sourceBuilder.bytes())); + builder.storeScript(randomAsciiOfLength(i + 1), + StoredScriptSource.parse(lang, sourceBuilder.bytes(), sourceBuilder.contentType())); } return builder.build(); } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 71fdea972f564..3482dc8bb348b 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; @@ -413,14 +414,15 @@ public void testStoreScript() throws Exception { .field("script", "abc") .endObject().bytes(); - ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id", StoredScriptSource.parse("_lang", script)); + ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id", + StoredScriptSource.parse("_lang", script, XContentType.JSON)); assertNotNull(scriptMetaData); assertEquals("abc", scriptMetaData.getStoredScript("_id", "_lang").getCode()); } public void testDeleteScript() throws Exception { ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id", - StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"))); + StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON)); scriptMetaData = ScriptMetaData.deleteStoredScript(scriptMetaData, "_id", "_lang"); assertNotNull(scriptMetaData); assertNull(scriptMetaData.getStoredScript("_id", "_lang")); @@ -438,7 +440,7 @@ public void testGetStoredScript() throws Exception { .metaData(MetaData.builder() .putCustom(ScriptMetaData.TYPE, new ScriptMetaData.Builder(null).storeScript("_id", - StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"))).build())) + StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON)).build())) .build(); assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")).getCode()); diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java index 8a6d54c0b9cf8..5f000f28cbcf1 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java @@ -187,7 +187,7 @@ public void testSourceParsing() throws Exception { try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { builder.startObject().field("script", "code").endObject(); - StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap()); assertThat(parsed, equalTo(source)); @@ -197,7 +197,7 @@ public void testSourceParsing() throws Exception { try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { builder.startObject().field("template", "code").endObject(); - StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType())); @@ -213,7 +213,7 @@ public void testSourceParsing() throws Exception { code = cb.startObject().field("query", "code").endObject().string(); } - StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", code, Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType())); @@ -229,7 +229,7 @@ public void testSourceParsing() throws Exception { code = cb.startObject().field("query", "code").endObject().string(); } - StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", code, Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType())); @@ -245,7 +245,7 @@ public void testSourceParsing() throws Exception { code = cb.startObject().field("query", "code").endObject().string(); } - StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", code, Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType())); @@ -256,7 +256,7 @@ public void testSourceParsing() throws Exception { try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code").endObject().endObject(); - StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap()); assertThat(parsed, equalTo(source)); @@ -267,7 +267,7 @@ public void testSourceParsing() throws Exception { builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code") .field("options").startObject().endObject().endObject().endObject(); - StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap()); assertThat(parsed, equalTo(source)); @@ -283,7 +283,7 @@ public void testSourceParsing() throws Exception { code = cb.startObject().field("query", "code").endObject().string(); } - StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes()); + StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("lang", code, Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType())); @@ -297,7 +297,7 @@ public void testSourceParsingErrors() throws Exception { builder.startObject().field("template", "code").endObject(); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> - StoredScriptSource.parse(null, builder.bytes())); + StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON)); assertThat(iae.getMessage(), equalTo("unexpected stored script format")); } @@ -306,7 +306,7 @@ public void testSourceParsingErrors() throws Exception { builder.startObject().field("script").startObject().field("code", "code").endObject().endObject(); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> - StoredScriptSource.parse(null, builder.bytes())); + StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON)); assertThat(iae.getMessage(), equalTo("must specify lang for stored script")); } @@ -315,7 +315,7 @@ public void testSourceParsingErrors() throws Exception { builder.startObject().field("script").startObject().field("lang", "lang").endObject().endObject(); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> - StoredScriptSource.parse(null, builder.bytes())); + StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON)); assertThat(iae.getMessage(), equalTo("must specify code for stored script")); } @@ -325,7 +325,7 @@ public void testSourceParsingErrors() throws Exception { .startObject("options").field("option", "option").endObject().endObject().endObject(); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> - StoredScriptSource.parse(null, builder.bytes())); + StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON)); assertThat(iae.getMessage(), equalTo("illegal compiler options [{option=option}] specified")); } @@ -335,7 +335,7 @@ public void testSourceParsingErrors() throws Exception { .startObject("options").field("content_type", "option").endObject().endObject().endObject(); ParsingException pe = expectThrows(ParsingException.class, () -> - StoredScriptSource.parse(null, builder.bytes())); + StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON)); assertThat(pe.getRootCause().getMessage(), equalTo("content_type cannot be user-specified")); } } diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java index 106cb82526fae..f5c70edb0246f 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -52,7 +53,7 @@ public void testBasics() { assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(LANG) .setId("foobar") - .setContent(new BytesArray("{\"script\":\"1\"}"))); + .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON)); String script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar") .get().getSource().getCode(); assertNotNull(script); @@ -68,7 +69,7 @@ public void testBasics() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript() .setLang("lang#") .setId("id#") - .setContent(new BytesArray("{}")) + .setContent(new BytesArray("{}"), XContentType.JSON) .get()); assertEquals("Validation Failed: 1: id cannot contain '#' for stored script;" + "2: lang cannot contain '#' for stored script;", e.getMessage()); @@ -78,7 +79,7 @@ public void testMaxScriptSize() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript() .setLang(LANG) .setId("foobar") - .setContent(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1))) + .setContent(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)), XContentType.JSON) .get() ); assertEquals("exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", e.getMessage()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index d57e2774b55c1..2268826100478 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -22,8 +22,10 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.children.Children; @@ -121,8 +123,8 @@ public void setupSuiteScopeCluster() throws Exception { requests.add(client().prepareIndex("test", "article", "b").setSource("category", new String[]{"a", "b"}, "randomized", false)); requests.add(client().prepareIndex("test", "article", "c").setSource("category", new String[]{"a", "b", "c"}, "randomized", false)); requests.add(client().prepareIndex("test", "article", "d").setSource("category", new String[]{"c"}, "randomized", false)); - requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}")); - requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}")); + requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}", XContentType.JSON)); indexRandom(true, requests); ensureSearchable("test"); @@ -240,7 +242,7 @@ public void testWithDeletes() throws Exception { ); List requests = new ArrayList<>(); - requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}")); + requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex(indexName, "child", "0").setParent("1").setSource("count", 1)); requests.add(client().prepareIndex(indexName, "child", "1").setParent("1").setSource("count", 1)); requests.add(client().prepareIndex(indexName, "child", "2").setParent("1").setSource("count", 1)); @@ -267,7 +269,7 @@ public void testWithDeletes() throws Exception { */ UpdateResponse updateResponse = client().prepareUpdate(indexName, "child", idToUpdate) .setParent("1") - .setDoc("count", 1) + .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1) .setDetectNoop(false) .get(); assertThat(updateResponse.getVersion(), greaterThan(1L)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index 51d702e3548b9..522981b4f01af 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -397,8 +398,8 @@ public void testParentFilterResolvedCorrectly() throws Exception { ensureGreen("idx2"); List indexRequests = new ArrayList<>(2); - indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}")); - indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}")); + indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}", XContentType.JSON)); + indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}", XContentType.JSON)); indexRandom(true, indexRequests); SearchResponse response = client().prepareSearch("idx2").setTypes("provider") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index c8cd235aedec2..4a8c88b3093fe 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardException; @@ -305,7 +306,7 @@ public void testXContentResponse() throws Exception { public void testDeletesIssue7951() throws Exception { String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; - assertAcked(prepareCreate(INDEX_NAME).setSettings(settings) + assertAcked(prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON) .addMapping("doc", "text", "type=keyword", CLASS_FIELD, "type=keyword")); String[] cat1v1 = {"constant", "one"}; String[] cat1v2 = {"constant", "uno"}; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index 764ed3ba73b09..f7e3d9a61b598 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; @@ -99,10 +100,10 @@ private void addTermsDocs(String term, int numInClass, int numNotInClass, List builders) { String sourceClass = "{\"text\": \"" + term + "\"}"; for (int i = 0; i < numDocs; i++) { - builders.add(client().prepareIndex(index, type).setSource(sourceClass)); + builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON)); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index ca88b56cd4257..f431cb0a47822 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; @@ -230,22 +231,22 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(CustomScriptPlugin.NAME) .setId("initScript_stored") - .setContent(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"))); + .setContent(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(CustomScriptPlugin.NAME) .setId("mapScript_stored") - .setContent(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"))); + .setContent(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(CustomScriptPlugin.NAME) .setId("combineScript_stored") - .setContent(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"))); + .setContent(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(CustomScriptPlugin.NAME) .setId("reduceScript_stored") - .setContent(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"))); + .setContent(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"), XContentType.JSON)); indexRandom(true, builders); ensureSearchable(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java index e23684532961e..5b8ea2d5c67d9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; @@ -482,7 +483,7 @@ public void testStoredScript() { .setId("my_script") .setLang(CustomScriptPlugin.NAME) // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin - .setContent(new BytesArray("{ \"script\": \"my_script\" }"))); + .setContent(new BytesArray("{ \"script\": \"my_script\" }"), XContentType.JSON)); SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java index 689eafdf5a500..68904c94ca396 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; @@ -434,7 +435,8 @@ public void testStoredScript() { .setId("my_script") .setLang(CustomScriptPlugin.NAME) // Source is not interpreted but my_script is defined in CustomScriptPlugin - .setContent(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"))); + .setContent(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"), + XContentType.JSON)); Script script = new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "my_script", Collections.emptyMap()); diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index a4096c0e34dbf..ef1750a448c1e 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -280,7 +280,7 @@ public void testToXContent() throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = builder.bytes(); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); + Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); assertEquals(0, sourceAsMap.size()); } { @@ -289,7 +289,7 @@ public void testToXContent() throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = builder.bytes(); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); + Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); assertEquals(1, sourceAsMap.size()); assertEquals("query", sourceAsMap.keySet().iterator().next()); } diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index 391d8be412e4f..78cb846932f38 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.HasChildQueryBuilder; @@ -1009,7 +1010,7 @@ public void testParentFieldQuery() throws Exception { .get(); assertHitCount(response, 0L); - client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get(); + client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get(); refresh(); response = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).get(); @@ -1018,7 +1019,7 @@ public void testParentFieldQuery() throws Exception { response = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).get(); assertHitCount(response, 1L); - client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get(); + client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get(); refresh(); response = client().prepareSearch("test").setQuery(termsQuery("_parent#parent", "p1", "p2")).get(); assertHitCount(response, 2L); @@ -1039,13 +1040,13 @@ public void testParentIdQuery() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get(); + client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get(); refresh(); SearchResponse response = client().prepareSearch("test").setQuery(parentId("child", "p1")).get(); assertHitCount(response, 1L); - client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get(); + client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get(); refresh(); response = client().prepareSearch("test") @@ -1405,8 +1406,8 @@ public void testParentChildQueriesViaScrollApi() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); for (int i = 0; i < 10; i++) { - client().prepareIndex("test", "parent", "p" + i).setSource("{}").get(); - client().prepareIndex("test", "child", "c" + i).setSource("{}").setParent("p" + i).get(); + client().prepareIndex("test", "parent", "p" + i).setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "c" + i).setSource("{}", XContentType.JSON).setParent("p" + i).get(); } refresh(); @@ -1488,8 +1489,8 @@ public void testTypeIsAppliedInHasParentInnerQuery() throws Exception { List indexRequests = new ArrayList<>(); indexRequests.add(client().prepareIndex("test", "parent", "1").setSource("field1", "a")); - indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}")); - indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}")); + indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON)); + indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}", XContentType.JSON)); indexRandom(true, indexRequests); SearchResponse searchResponse = client().prepareSearch("test") @@ -1883,8 +1884,8 @@ public void testMinMaxChildren() throws Exception { public void testParentFieldToNonExistingType() { assertAcked(prepareCreate("test").addMapping("parent").addMapping("child", "_parent", "type=parent2")); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); try { @@ -1898,8 +1899,8 @@ public void testParentFieldToNonExistingType() { public void testHasParentInnerQueryType() { assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type")); - client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get(); - client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get(); + client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get(); refresh(); //make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead SearchResponse searchResponse = client().prepareSearch("test").setTypes("child-type").setQuery( @@ -1909,8 +1910,8 @@ public void testHasParentInnerQueryType() { public void testHasChildInnerQueryType() { assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type")); - client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get(); - client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get(); + client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get(); refresh(); //make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead SearchResponse searchResponse = client().prepareSearch("test").setTypes("parent-type").setQuery( diff --git a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java index 5aead911caa57..44b01b3a4b974 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -69,8 +70,8 @@ public void testEagerParentFieldLoading() throws Exception { .addMapping("child", childMapping(false))); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); @@ -84,8 +85,8 @@ public void testEagerParentFieldLoading() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); @@ -101,9 +102,9 @@ public void testEagerParentFieldLoading() throws Exception { // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals // is loaded by the size of the field data cache, because global ordinals on 1 segment shards takes no extra memory. - client().prepareIndex("test", "parent", "1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); refresh(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); @@ -117,8 +118,8 @@ public void testChangingEagerParentFieldLoadingAtRuntime() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); @@ -153,7 +154,7 @@ public void run() { // Need to add a new doc otherwise the refresh doesn't trigger a new searcher // Because it ends up in its own segment, but isn't of type parent or child, this doc doesn't contribute to the size of the fielddata cache - client().prepareIndex("test", "dummy", "dummy").setSource("{}").get(); + client().prepareIndex("test", "dummy", "dummy").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index c00882522243a..1d24a1f16d5b9 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -331,19 +332,19 @@ public void testRandomParentChild() throws Exception { int[] child2InnerObjects = new int[numDocs]; for (int parent = 0; parent < numDocs; parent++) { String parentId = String.format(Locale.ENGLISH, "%03d", parent); - requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}")); + requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}", XContentType.JSON)); int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs); int limit = child1 + numChildDocs; for (; child1 < limit; child1++) { requestBuilders.add(client().prepareIndex("idx", "child1", - String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}")); + String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}", XContentType.JSON)); } numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs); limit = child2 + numChildDocs; for (; child2 < limit; child2++) { requestBuilders.add(client().prepareIndex("idx", "child2", - String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}")); + String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}", XContentType.JSON)); } } indexRandom(true, requestBuilders); @@ -727,17 +728,26 @@ public void testRoyals() throws Exception { ); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("royals", "king", "king").setSource("{}")); - requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king").setSource("{}")); + requests.add(client().prepareIndex("royals", "king", "king").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king") + .setSource("{}", XContentType.JSON)); indexRandom(true, requests); SearchResponse response = client().prepareSearch("royals") @@ -899,10 +909,10 @@ public void testMatchesQueriesNestedInnerHits() throws Exception { public void testMatchesQueriesParentChildInnerHits() throws Exception { assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("index", "parent", "1").setSource("{}")); + requests.add(client().prepareIndex("index", "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index", "child", "1").setParent("1").setSource("field", "value1")); requests.add(client().prepareIndex("index", "child", "2").setParent("1").setSource("field", "value2")); - requests.add(client().prepareIndex("index", "parent", "2").setSource("{}")); + requests.add(client().prepareIndex("index", "parent", "2").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index", "child", "3").setParent("2").setSource("field", "value1")); indexRandom(true, requests); @@ -938,7 +948,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { public void testDontExplode() throws Exception { assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}")); + requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index1", "child", "1").setParent("1").setSource("field", "value1")); indexRandom(true, requests); @@ -1025,7 +1035,7 @@ public void testInnerHitsWithIgnoreUnmapped() throws Exception { ); createIndex("index2"); client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get(); - client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}").get(); + client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}", XContentType.JSON).get(); client().prepareIndex("index2", "type", "3").setSource("key", "value").get(); refresh(); diff --git a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 236d5ce1ab8ef..a952ae8639695 100644 --- a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; @@ -689,8 +690,8 @@ public void testGetFieldsComplexField() throws Exception { .endArray() .endObject().bytes(); - client().prepareIndex("my-index", "my-type1", "1").setSource(source).get(); - client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source).get(); + client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get(); + client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source, XContentType.JSON).get(); String field = "field1.field2.field3.field4"; diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index 2707e3c0b275c..568c723f8d16e 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -47,6 +47,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; @@ -230,7 +231,7 @@ public void testShapeRelations() throws Exception { new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close())); BytesReference data = jsonBuilder().startObject().field("area", polygon).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // Point in polygon @@ -292,7 +293,7 @@ public void testShapeRelations() throws Exception { new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close())); data = jsonBuilder().startObject().field("area", inverse).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "2").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "2").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // re-check point on polygon hole @@ -326,7 +327,7 @@ public void testShapeRelations() throws Exception { .coordinate(170, -10).coordinate(190, -10).coordinate(190, 10).coordinate(170, 10).close()); data = jsonBuilder().startObject().field("area", builder).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // Create a polygon crossing longitude 180 with hole. @@ -335,7 +336,7 @@ public void testShapeRelations() throws Exception { .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(175, -5).coordinate(185, -5).coordinate(185, 5).coordinate(175, 5).close())); data = jsonBuilder().startObject().field("area", builder).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); result = client().prepareSearch() @@ -384,7 +385,7 @@ public void testBulk() throws Exception { client().admin().indices().prepareCreate("countries").setSettings(settings) .addMapping("country", xContentBuilder.string()).execute().actionGet(); - BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null).execute().actionGet(); + BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null, xContentBuilder.contentType()).get(); for (BulkItemResponse item : bulk.getItems()) { assertFalse("unable to index data", item.isFailed()); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 067c2dcac0556..7e8aa1ad9d0fe 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.locationtech.spatial4j.shape.Rectangle; import com.vividsolutions.jts.geom.Coordinate; @@ -69,7 +70,8 @@ public void testNullShape() throws Exception { client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); - client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}", XContentType.JSON) + .setRefreshPolicy(IMMEDIATE).get(); GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet(); assertThat(result.getField("location"), nullValue()); } @@ -252,8 +254,8 @@ public void testShapeFetchingPath() throws Exception { .setSource( String.format( Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location - ) - ).setRefreshPolicy(IMMEDIATE).get(); + ), XContentType.JSON) + .setRefreshPolicy(IMMEDIATE).get(); client().prepareIndex("test", "type", "1") .setSource(jsonBuilder().startObject().startObject("location") .field("type", "polygon") diff --git a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java index ccc3102f3a80b..c7e78f2cc28fd 100644 --- a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -59,7 +60,7 @@ public void testSimpleMultiSearchMoreRequests() { createIndex("test"); int numDocs = randomIntBetween(0, 16); for (int i = 0; i < numDocs; i++) { - client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get(); + client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}", XContentType.JSON).get(); } refresh(); diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 2914755ed73c7..4b3b890a20d70 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -98,7 +99,7 @@ public void testNoPreferenceRandom() throws Exception { } public void testSimplePreference() throws Exception { - client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}").get(); + client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen(); client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); @@ -131,7 +132,7 @@ public void testSimplePreference() throws Exception { } public void testReplicaPreference() throws Exception { - client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}").get(); + client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}", XContentType.JSON).get(); ensureGreen(); client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); @@ -147,7 +148,7 @@ public void testReplicaPreference() throws Exception { SearchResponse resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet(); assertThat(resp.getHits().totalHits(), equalTo(1L)); - client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}").get(); + client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen("test"); resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java index 2fe14e4ee102b..1dbeb24d4ed9e 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; @@ -64,7 +65,7 @@ protected Collection> nodePlugins() { @Before public void setup() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); - prepareCreate("test").setSource(indexBody).get(); + prepareCreate("test").setSource(indexBody, XContentType.JSON).get(); ensureGreen("test"); } @@ -151,7 +152,7 @@ public void testWithLotsOfTypes() throws Exception { public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); - reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody)); + reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON)); indexRandom(true, false, reqs); SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get(); @@ -214,7 +215,7 @@ public void testKeywordWithWhitespace() throws Exception { public void testExplicitAllFieldsRequested() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json"); - prepareCreate("test2").setSource(indexBody).get(); + prepareCreate("test2").setSource(indexBody, XContentType.JSON).get(); ensureGreen("test2"); List reqs = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index ebc2df30e1d3e..7cb9627cf1835 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; @@ -1796,8 +1797,8 @@ public void testRangeQueryWithTimeZone() throws Exception { } public void testSearchEmptyDoc() { - assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}")); - client().prepareIndex("test", "type1", "1").setSource("{}").get(); + assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}", XContentType.JSON)); + client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON).get(); refresh(); assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 3ed464f312c1e..1bec4bb20bfbe 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; @@ -471,7 +472,7 @@ public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); - reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody)); + reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON)); indexRandom(true, false, reqs); SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get(); diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java index df89bda88d83b..ae8ad66ac8d94 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.search.RestClearScrollAction; @@ -48,7 +49,8 @@ public void testParseClearScrollRequest() throws Exception { public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception { RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java index 7d2a5024cc8de..662bc07f90d7e 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.search.RestSearchScrollAction; @@ -52,7 +53,8 @@ public void testParseSearchScrollRequest() throws Exception { public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception { RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 3927d87912901..1f368cb35daf4 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -521,7 +521,7 @@ public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException { private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); response.toXContent(builder, ToXContent.EMPTY_PARAMS); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map.get("succeeded"), is(succeed)); assertThat(map.get("num_freed"), equalTo(numFreed)); } diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 099cce038c116..f63f13b6dd24b 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; @@ -289,7 +290,7 @@ public void testSimpleTerminateAfterCount() throws Exception { public void testInsaneFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); @@ -297,7 +298,7 @@ public void testInsaneFromAndSize() throws Exception { public void testTooLargeFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); @@ -307,7 +308,7 @@ public void testTooLargeFromAndSize() throws Exception { public void testLargeFromAndSizeSucceeds() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); @@ -318,7 +319,7 @@ public void testLargeFromAndSizeSucceeds() throws Exception { public void testTooLargeFromAndSizeOkBySetting() throws Exception { prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); @@ -333,7 +334,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); @@ -343,7 +344,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); @@ -353,7 +354,7 @@ public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws public void testTooLargeRescoreWindow() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertRescoreWindowFails(Integer.MAX_VALUE); assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); @@ -363,7 +364,7 @@ public void testTooLargeRescoreOkBySetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); prepareCreate("idx").setSettings(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -374,7 +375,7 @@ public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. defaultMaxWindow * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -388,7 +389,7 @@ public void testTooLargeRescoreOkByDynamicSetting() throws Exception { .setSettings( Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -403,7 +404,7 @@ public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception // Note that this is the RESULT window Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index 84dd3dabf6b61..7659576555868 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; @@ -91,7 +92,7 @@ public void testIssue8226() { assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test"))); } if (i > 0) { - client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get(); + client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}", XContentType.JSON).get(); } } refresh(); @@ -422,9 +423,9 @@ public void testIssue2986() { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=keyword").get()); - client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet(); - client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet(); - client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet(); + client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}", XContentType.JSON).execute().actionGet(); + client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}", XContentType.JSON).execute().actionGet(); + client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}", XContentType.JSON).execute().actionGet(); refresh(); SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true) .addSort("field1", SortOrder.ASC).execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 0000e73e27dc9..6116cfa8b98c1 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; @@ -533,7 +534,7 @@ public void testIncludeGlobalState() throws Exception { .endObject() .endArray() .endObject().bytes(); - assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource).get()); + assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource, XContentType.JSON).get()); } if(testScript) { @@ -541,7 +542,7 @@ public void testIncludeGlobalState() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(MockScriptEngine.NAME) .setId("foobar") - .setContent(new BytesArray("{\"script\":\"1\"}"))); + .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON)); } logger.info("--> snapshot without global state"); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java index cdacabdd66a14..a8e1eeaec106b 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java @@ -79,7 +79,7 @@ public void testRestoreSnapshotRequestParsing() throws IOException { BytesReference bytes = builder.endObject().bytes(); - request.source(XContentHelper.convertToMap(bytes, true).v2()); + request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2()); assertEquals("test-repo", request.repository()); assertEquals("test-snap", request.snapshot()); @@ -137,7 +137,7 @@ public void testCreateSnapshotRequestParsing() throws IOException { BytesReference bytes = builder.endObject().bytes(); - request.source(XContentHelper.convertToMap(bytes, true).v2()); + request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2()); assertEquals("test-repo", request.repository()); assertEquals("test-snap", request.snapshot()); diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 7fc1490de73f6..6dd4fa384e99b 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; @@ -80,7 +81,7 @@ public static void index01Docs(String type, String settings, ESIntegTestCase tes if (type.equals("text")) { textMappings += ",fielddata=true"; } - assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings) + assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON) .addMapping("doc", "text", textMappings, CLASS_FIELD, "type=keyword")); String[] gb = {"0", "1"}; List indexRequestBuilderList = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java index 01fc15e6f75f5..33047b2d12a6a 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java @@ -78,7 +78,7 @@ public void testThatToXContentWritesOutUnboundedCorrectly() throws Exception { info.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map, hasKey("foo")); map = (Map) map.get("foo"); assertThat(map, hasKey("queue_size")); @@ -100,7 +100,7 @@ public void testThatToXContentWritesInteger() throws Exception { info.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map, hasKey("foo")); map = (Map) map.get("foo"); assertThat(map, hasKey("queue_size")); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 2c98b393642ab..bc0658232159e 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.DocumentMissingException; @@ -536,7 +537,7 @@ public void testVersionedUpdate() throws Exception { // With internal versions, tt means "if object is there with version X, update it or explode. If it is not there, index. client().prepareUpdate(indexOrAlias(), "type", "3") .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))) - .setVersion(10).setUpsert("{ \"text\": \"v0\" }").get(); + .setVersion(10).setUpsert("{ \"text\": \"v0\" }", XContentType.JSON).get(); get = get("test", "type", "3"); assertThat(get.getVersion(), equalTo(1L)); assertThat((String) get.getSource().get("text"), equalTo("v0")); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index f14d91465f6e2..beab15165d698 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -245,7 +245,7 @@ private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XCon private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1") - .setDoc(xContentBuilder.bytes().utf8ToString()) + .setDoc(xContentBuilder) .setDocAsUpsert(true) .setFields("_source"); if (detectNoop != null) { diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java index 71ec194866266..c61b736bf6db1 100644 --- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.rest; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; @@ -47,12 +48,14 @@ public void testIndexWithoutId() throws IOException { public void testUpsert() throws IOException { locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{" + "\"doc\": {\"test\": \"test\"}," - + "\"doc_as_upsert\": true}"))); + + "\"doc_as_upsert\": true}", ContentType.APPLICATION_JSON))); } private void locationTestCase(String method, String url) throws IOException { - locationTestCase(client().performRequest(method, url, emptyMap(), new StringEntity("{\"test\": \"test\"}"))); - locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), new StringEntity("{\"test\": \"test\"}"))); + locationTestCase(client().performRequest(method, url, emptyMap(), + new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON))); + locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), + new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON))); } private void locationTestCase(Response response) throws IOException { diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 712a7969a0517..7e8ed1c1c816f 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -670,7 +670,27 @@ The response looks like: For libraries that don't accept a request body for non-POST requests, you can pass the request body as the `source` query string parameter -instead. +instead. When using this method, the `source_content_type` parameter +should also be passed with a media type value that indicates the format +of the source, such as `application/json`. + +[float] +=== Content-Type auto-detection + +deprecated[5.3.0, Provide the proper Content-Type header] + +The content sent in a request body or using the `source` query string +parameter is inspected to automatically determine the content type +(JSON, YAML, SMILE, or CBOR). + +A strict mode can be enabled which disables auto-detection and requires +that all requests with a body have a Content-Type header that maps to +a supported format. To enabled this strict mode, add the following +setting to the `elasticsearch.yml` file: + + http.content_type.required: true + +The default value is `false`. [[url-access-control]] == URL-based access control @@ -682,7 +702,7 @@ the choice of specifying an index in the URL and on each individual request within the request body. This can make URL-based access control challenging. To prevent the user from overriding the index which has been specified in the -URL, add this setting to the `config.yml` file: +URL, add this setting to the `elasticsearch.yml` file: rest.action.multi.allow_explicit_index: false diff --git a/docs/reference/migration/migrate_6_0/rest.asciidoc b/docs/reference/migration/migrate_6_0/rest.asciidoc index 1999ff6517126..f7b52b914daf3 100644 --- a/docs/reference/migration/migrate_6_0/rest.asciidoc +++ b/docs/reference/migration/migrate_6_0/rest.asciidoc @@ -13,6 +13,12 @@ has been removed in Elasticsearch 6.0.0. In previous versions of Elasticsearch, documents were allowed to contain duplicate keys. Elasticsearch 6.0.0 enforces that all keys are unique. This applies to all content types: JSON, CBOR, Yaml and Smile. +==== Content-Type Auto-detection + +In previous versions of Elasticsearch, having a proper Content-Type for the data in a request was not enforced. +Elasticsearch 6.0.0 enforces that all requests with a body must have a supported Content-Type and this type will +be used when parsing the data. + ==== Boolean API parameters All REST APIs parameters (both request parameters and JSON body) support providing boolean "false" as the diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index 065c91349c25a..3682234f5e6fb 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -100,6 +100,9 @@ simple message will be returned. Defaults to `true` |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`. +|`http.content_type.required`|Enables or disables strict checking and usage of +the `Content-Type` header for all requests with content, defaults to `false`. + |======================================================================= It also uses the common diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java index e2e7b81933335..697dbf625752b 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -53,9 +54,9 @@ public void testAllOpsDisabledIndexedScripts() throws IOException { client().admin().cluster().preparePutStoredScript() .setLang(ExpressionScriptEngineService.NAME) .setId("script1") - .setContent(new BytesArray("{\"script\":\"2\"}")) + .setContent(new BytesArray("{\"script\":\"2\"}"), XContentType.JSON) .get(); - client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get(); + client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get(); try { client().prepareUpdate("test", "scriptTest", "1") .setScript(new Script(ScriptType.STORED, ExpressionScriptEngineService.NAME, "script1", Collections.emptyMap())).get(); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java index 6c22b0160b6dd..83925f0ec03b6 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java @@ -47,7 +47,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client String id = request.param("id"); BytesReference content = request.content(); - PutStoredScriptRequest put = new PutStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG, content); + PutStoredScriptRequest put = new PutStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG, content, request.getXContentType()); return channel -> client.admin().cluster().putStoredScript(put, new AcknowledgedRestListener<>(channel)); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java index 3e4471509eea2..7dad299a40f0e 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; @@ -34,7 +35,8 @@ public class MultiSearchTemplateRequestTests extends ESTestCase { public void testParseRequest() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/script/mustache/simple-msearch-template.json"); - RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(data)).build(); + RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray(data), XContentType.JSON).build(); MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index ba19febfd214d..2b11df19392a9 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -279,7 +279,8 @@ public void testEmbeddedToJSON() throws Exception { .endArray() .endObject(); - Map ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2()); + Map ctx = + Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2()); assertScript("{{#ctx.bulks}}{{#toJson}}.{{/toJson}}{{/ctx.bulks}}", ctx, equalTo("{\"index\":\"index-1\",\"id\":1,\"type\":\"type-1\"}{\"index\":\"index-2\",\"id\":2,\"type\":\"type-2\"}")); @@ -320,7 +321,8 @@ public void testEmbeddedArrayJoin() throws Exception { .endArray() .endObject(); - Map ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2()); + Map ctx = + Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2()); assertScript("{{#join}}ctx.people.0.emails{{/join}}", ctx, equalTo("john@smith.com,john.smith@email.com,jsmith@email.com")); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 8f6e77fc51519..19a702383369a 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptType; @@ -159,7 +160,7 @@ public void testIndexedTemplateClient() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}"))); + "}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() @@ -171,18 +172,18 @@ public void testIndexedTemplateClient() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}"))); + "}"), XContentType.JSON)); GetStoredScriptResponse getResponse = client().admin().cluster() .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get(); assertNotNull(getResponse.getSource()); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); @@ -221,7 +222,7 @@ public void testIndexedTemplate() throws Exception { " }" + "}" + "}" - )) + ), XContentType.JSON) ); assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(MustacheScriptEngineService.NAME) @@ -233,7 +234,7 @@ public void testIndexedTemplate() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}")) + "}"), XContentType.JSON) ); assertAcked(client().admin().cluster().preparePutStoredScript() .setLang(MustacheScriptEngineService.NAME) @@ -243,15 +244,15 @@ public void testIndexedTemplate() throws Exception { " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + - "}")) + "}"), XContentType.JSON) ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); @@ -315,7 +316,7 @@ public void testIndexedTemplateOverwrite() throws Exception { .setLang(MustacheScriptEngineService.NAME) .setId("git01") .setContent(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + - "\"type\": \"ooophrase_prefix\"}}}}}"))); + "\"type\": \"ooophrase_prefix\"}}}}}"), XContentType.JSON)); GetStoredScriptResponse getResponse = client().admin().cluster() .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get(); @@ -335,7 +336,7 @@ public void testIndexedTemplateOverwrite() throws Exception { .setLang(MustacheScriptEngineService.NAME) .setId("git01") .setContent(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + - "\"type\": \"phrase_prefix\"}}}}"))); + "\"type\": \"phrase_prefix\"}}}}"), XContentType.JSON)); SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()) .setRequest(new SearchRequest("testindex").types("test")) @@ -343,7 +344,6 @@ public void testIndexedTemplateOverwrite() throws Exception { .get(); assertHitCount(searchResponse.getResponse(), 1); assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"); - } } @@ -353,14 +353,14 @@ public void testIndexedTemplateWithArray() throws Exception { client().admin().cluster().preparePutStoredScript() .setLang(MustacheScriptEngineService.NAME) .setId("4") - .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes()) + .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON) ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 632c50228389f..97a65a29adab2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -57,6 +57,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.analysis.FieldNameAnalyzer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperForType; @@ -94,6 +95,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder pqb.toQuery(createShardContext())); assertThat(e.getMessage(), equalTo("query builder must be rewritten first")); QueryBuilder rewrite = pqb.rewrite(createShardContext()); - PercolateQueryBuilder geoShapeQueryBuilder = new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource); + PercolateQueryBuilder geoShapeQueryBuilder = + new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource, XContentType.JSON); assertEquals(geoShapeQueryBuilder, rewrite); } @@ -176,14 +182,16 @@ protected Set getObjectsHoldingArbitraryContent() { public void testRequiredParameters() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - new PercolateQueryBuilder(null, null, new BytesArray("{}")); + new PercolateQueryBuilder(null, null, new BytesArray("{}"), XContentType.JSON); }); assertThat(e.getMessage(), equalTo("[field] is a required argument")); - e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"))); + e = expectThrows(IllegalArgumentException.class, + () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"), XContentType.JSON)); assertThat(e.getMessage(), equalTo("[document_type] is a required argument")); - e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", "_document_type", null)); + e = expectThrows(IllegalArgumentException.class, + () -> new PercolateQueryBuilder("_field", "_document_type", null, null)); assertThat(e.getMessage(), equalTo("[document] is a required argument")); e = expectThrows(IllegalArgumentException.class, () -> { @@ -226,7 +234,7 @@ public void testCreateMultiDocumentSearcher() throws Exception { } Analyzer analyzer = new WhitespaceAnalyzer(); - ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null); + ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null, null); IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs)); @@ -239,6 +247,26 @@ public void testCreateMultiDocumentSearcher() throws Exception { assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT)); } + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("P4AAAAAFZmllbGQEdHlwZQAAAAAAAA57ImZvbyI6ImJhciJ9AAAAAA=="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(in); + assertEquals("type", queryBuilder.getDocumentType()); + assertEquals("field", queryBuilder.getField()); + assertEquals("{\"foo\":\"bar\"}", queryBuilder.getDocument().utf8ToString()); + assertEquals(XContentType.JSON, queryBuilder.getXContentType()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + queryBuilder.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } + private static BytesReference randomSource() { try { XContentBuilder xContent = XContentFactory.jsonBuilder(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 4d94ab3a29611..d47e0a461443c 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -565,7 +565,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception { ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() - .rawField(fieldName, new BytesArray(query.string())) + .rawField(fieldName, new BytesArray(query.string()), query.contentType()) .endObject().bytes()); BytesRef querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); Map parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); @@ -592,7 +592,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception { doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() - .rawField(fieldName, new BytesArray(query.string())) + .rawField(fieldName, new BytesArray(query.string()), query.contentType()) .endObject().bytes()); querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 1cce5fdbbdbf2..07fc9aa2fe66f 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; @@ -94,7 +95,8 @@ public void testPercolateScriptQuery() throws IOException { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .execute().actionGet(); SearchResponse response = client().prepareSearch("index") - .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes())) + .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), + XContentType.JSON)) .get(); assertHitCount(response, 1); assertSearchHits(response, "1"); @@ -122,7 +124,7 @@ public void testPercolatorQuery() throws Exception { BytesReference source = jsonBuilder().startObject().endObject().bytes(); logger.info("percolating empty doc"); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -130,7 +132,7 @@ public void testPercolatorQuery() throws Exception { source = jsonBuilder().startObject().field("field1", "value").endObject().bytes(); logger.info("percolating doc with 1 field"); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 2); @@ -140,7 +142,7 @@ public void testPercolatorQuery() throws Exception { source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes(); logger.info("percolating doc with 2 fields"); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 3); @@ -199,7 +201,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test long range: BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("3")); @@ -207,7 +209,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field1", 11).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -215,7 +217,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test double range: source = jsonBuilder().startObject().field("field2", 12).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("6")); @@ -223,7 +225,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field2", 11).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("4")); @@ -231,7 +233,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test IP range: source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("9")); @@ -239,7 +241,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("7")); @@ -263,7 +265,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { .must(matchQuery("field2", "value")) ).endObject()).get(); - client().prepareIndex("test", "type", "1").setSource("{}").get(); + client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get(); client().prepareIndex("test", "type", "2").setSource("field1", "value").get(); client().prepareIndex("test", "type", "3").setSource("field1", "value", "field2", "value").get(); client().admin().indices().prepareRefresh().get(); @@ -305,7 +307,7 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) .get(); - client().prepareIndex("test", "type", "1").setSource("{}").get(); + client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get(); client().admin().indices().prepareRefresh().get(); logger.info("percolating empty doc with source disabled"); @@ -376,7 +378,7 @@ public void testPercolatorSpecificQueries() throws Exception { .field("field2", "the quick brown fox falls down into the well") .endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 4); @@ -423,7 +425,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { .field("field1", "The quick brown fox jumps over the lazy dog") .endObject().bytes(); SearchResponse searchResponse = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", document)) + .setQuery(new PercolateQueryBuilder("query", "type", document, XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) .addSort("_uid", SortOrder.ASC) .get(); @@ -457,7 +459,7 @@ public void testTakePositionOffsetGapIntoAccount() throws Exception { client().admin().indices().prepareRefresh().get(); SearchResponse response = client().prepareSearch().setQuery( - new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}")) + new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"), XContentType.JSON) ).get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); @@ -517,7 +519,7 @@ public void testWithMultiplePercolatorFields() throws Exception { BytesReference source = jsonBuilder().startObject().field("field", "value").endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source)) + .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source, XContentType.JSON)) .setIndices("test1") .get(); assertHitCount(response, 1); @@ -526,7 +528,7 @@ public void testWithMultiplePercolatorFields() throws Exception { assertThat(response.getHits().getAt(0).index(), equalTo("test1")); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source)) + .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source, XContentType.JSON)) .setIndices("test2") .get(); assertHitCount(response, 1); @@ -576,7 +578,7 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { .startObject().field("name", "virginia potts").endObject() .startObject().field("name", "tony stark").endObject() .endArray() - .endObject().bytes())) + .endObject().bytes(), XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 1); @@ -590,14 +592,15 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { .startObject().field("name", "virginia stark").endObject() .startObject().field("name", "tony stark").endObject() .endArray() - .endObject().bytes())) + .endObject().bytes(), XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 0); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "employee", - XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes())) + XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes(), + XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 0); @@ -632,16 +635,16 @@ public void testPercolatorQueryViaMultiSearch() throws Exception { MultiSearchResponse response = client().prepareMultiSearch() .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - jsonBuilder().startObject().field("field1", "b").endObject().bytes()))) + jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - yamlBuilder().startObject().field("field1", "c").endObject().bytes()))) + yamlBuilder().startObject().field("field1", "c").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - smileBuilder().startObject().field("field1", "b c").endObject().bytes()))) + smileBuilder().startObject().field("field1", "b c").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - jsonBuilder().startObject().field("field1", "d").endObject().bytes()))) + jsonBuilder().startObject().field("field1", "d").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null))) .add(client().prepareSearch("test") // non existing doc, so error element diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 03e4960cc964d..aa0dcd196177b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -62,6 +62,10 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.reindex.remote.RemoteInfo; @@ -72,6 +76,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -321,7 +327,24 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) // id and source always come from the found doc. Scripts can change them but they operate on the index request. index.id(doc.getId()); - index.source(doc.getSource()); + + // the source xcontent type and destination could be different + final XContentType sourceXContentType = doc.getXContentType(); + final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); + if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { + // we need to convert + try (XContentParser parser = sourceXContentType.xContent().createParser(NamedXContentRegistry.EMPTY, doc.getSource()); + XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent())) { + parser.nextToken(); + builder.copyCurrentStructure(parser); + index.source(builder.bytes(), builder.contentType()); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert hit from " + sourceXContentType + " to " + + mainRequestXContentType, e); + } + } else { + index.source(doc.getSource(), doc.getXContentType()); + } /* * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass @@ -329,7 +352,6 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) */ index.routing(mainRequest.getDestination().routing()); index.parent(mainRequest.getDestination().parent()); - index.contentType(mainRequest.getDestination().getContentType()); index.setPipeline(mainRequest.getDestination().getPipeline()); // OpType is synthesized from version so it is handled when we copy version above. diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java index 8e61c14c6c57f..a6afa6df39d77 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java @@ -172,6 +172,22 @@ static Map scrollParams(TimeValue keepAlive) { } static HttpEntity scrollEntity(String scroll) { - return new StringEntity(scroll, ContentType.TEXT_PLAIN); + try (XContentBuilder entity = JsonXContent.contentBuilder()) { + return new StringEntity(entity.startObject() + .field("scroll_id", scroll) + .endObject().string(), ContentType.APPLICATION_JSON); + } catch (IOException e) { + throw new ElasticsearchException("failed to build scroll entity", e); + } + } + + static HttpEntity clearScrollEntity(String scroll) { + try (XContentBuilder entity = JsonXContent.contentBuilder()) { + return new StringEntity(entity.startObject() + .array("scroll_id", scroll) + .endObject().string(), ContentType.APPLICATION_JSON); + } catch (IOException e) { + throw new ElasticsearchException("failed to build clear scroll entity", e); + } } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java index 31bcb3a8250f2..e9807bdfa5b81 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; @@ -33,7 +34,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.List; @@ -54,7 +55,7 @@ private RemoteResponseParsers() {} /** * Parser for an individual {@code hit} element. */ - public static final ConstructingObjectParser HIT_PARSER = + public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { int i = 0; String index = (String) a[i++]; @@ -68,15 +69,16 @@ private RemoteResponseParsers() {} HIT_PARSER.declareString(constructorArg(), new ParseField("_type")); HIT_PARSER.declareString(constructorArg(), new ParseField("_id")); HIT_PARSER.declareLong(optionalConstructorArg(), new ParseField("_version")); - HIT_PARSER.declareObject(BasicHit::setSource, (p, s) -> { + HIT_PARSER.declareObject(((basicHit, tuple) -> basicHit.setSource(tuple.v1(), tuple.v2())), (p, s) -> { try { /* * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for * now this should do. */ - try (XContentBuilder b = JsonXContent.contentBuilder()) { + try (XContentBuilder b = XContentBuilder.builder(s.xContent())) { b.copyCurrentStructure(p); - return b.bytes(); + // a hack but this lets us get the right xcontent type to go with the source + return new Tuple<>(b.bytes(), s); } } catch (IOException e) { throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e); @@ -92,7 +94,7 @@ class Fields { String routing; String parent; } - ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); + ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); hit.setParent(fields.parent); @@ -105,7 +107,7 @@ class Fields { /** * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}. */ - public static final ConstructingObjectParser HITS_PARSER = + public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); static { HITS_PARSER.declareLong(constructorArg(), new ParseField("total")); @@ -115,7 +117,7 @@ class Fields { /** * Parser for {@code failed} shards in the {@code _shards} elements. */ - public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = + public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = new ConstructingObjectParser<>("failure", true, a -> { int i = 0; String index = (String) a[i++]; @@ -148,7 +150,7 @@ class Fields { * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it * parses to an empty list. */ - public static final ConstructingObjectParser, Void> SHARDS_PARSER = + public static final ConstructingObjectParser, XContentType> SHARDS_PARSER = new ConstructingObjectParser<>("_shards", true, a -> { @SuppressWarnings("unchecked") List failures = (List) a[0]; @@ -159,7 +161,7 @@ class Fields { SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures")); } - public static final ConstructingObjectParser RESPONSE_PARSER = + public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>("search_response", true, a -> { int i = 0; Throwable catastrophicFailure = (Throwable) a[i++]; @@ -198,9 +200,9 @@ class Fields { * Collects stuff about Throwables and attempts to rebuild them. */ public static class ThrowableBuilder { - public static final BiFunction PARSER; + public static final BiFunction PARSER; static { - ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); + ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); PARSER = parser.andThen(ThrowableBuilder::build); parser.declareString(ThrowableBuilder::setType, new ParseField("type")); parser.declareString(ThrowableBuilder::setReason, new ParseField("reason")); @@ -219,7 +221,7 @@ public static class ThrowableBuilder { public Throwable build() { Throwable t = buildWithoutCause(); - if (causedBy != null) { + if (causedBy != null) { t.initCause(causedBy); } return t; @@ -264,10 +266,10 @@ public void setCausedBy(Throwable causedBy) { /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ - public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( "/", true, a -> (Version) a[0]); static { - ConstructingObjectParser versionParser = new ConstructingObjectParser<>( + ConstructingObjectParser versionParser = new ConstructingObjectParser<>( "version", true, a -> Version.fromString((String) a[0])); versionParser.declareString(constructorArg(), new ParseField("number")); MAIN_ACTION_PARSER.declareObject(constructorArg(), versionParser, new ParseField("version")); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java index 5b0785fc0c65f..9c78d14c69e5c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -21,6 +21,7 @@ import org.apache.http.ContentTooLongException; import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -57,6 +58,7 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; +import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.clearScrollEntity; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath; @@ -110,7 +112,7 @@ protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Cons @Override protected void clearScroll(String scrollId, Runnable onCompletion) { - client.performRequestAsync("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() { + client.performRequestAsync("DELETE", scrollPath(), emptyMap(), clearScrollEntity(scrollId), new ResponseListener() { @Override public void onSuccess(org.elasticsearch.client.Response response) { logger.debug("Successfully cleared [{}]", scrollId); @@ -140,7 +142,7 @@ protected void cleanup() { } private void execute(String method, String uri, Map params, HttpEntity entity, - BiFunction parser, Consumer listener) { + BiFunction parser, Consumer listener) { // Preserve the thread context so headers survive after the call java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); class RetryHelper extends AbstractRunnable { @@ -160,7 +162,8 @@ public void onSuccess(org.elasticsearch.client.Response response) { InputStream content = responseEntity.getContent(); XContentType xContentType = null; if (responseEntity.getContentType() != null) { - xContentType = XContentType.fromMediaTypeOrFormat(responseEntity.getContentType().getValue()); + final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType(); + xContentType = XContentType.fromMediaType(mimeType); } if (xContentType == null) { try { @@ -175,7 +178,7 @@ public void onSuccess(org.elasticsearch.client.Response response) { // EMPTY is safe here because we don't call namedObject try (XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) { - parsedResponse = parser.apply(xContentParser, null); + parsedResponse = parser.apply(xContentParser, xContentType); } catch (ParsingException e) { /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it * is totally wrong and we're probably not talking to Elasticsearch. */ diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index 5e5716d6b1486..01c5977e8223d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; @@ -204,7 +205,7 @@ public void testUpdateByQueryCancel() throws Exception { " \"test\" : {}\n" + " } ]\n" + "}"); - assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get()); + assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX), (response, total, modified) -> { assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request"))); @@ -237,7 +238,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { " \"test\" : {}\n" + " } ]\n" + "}"); - assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get()); + assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5), (response, total, modified) -> { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java index abef7fb590294..b190aee3619a8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java @@ -143,7 +143,7 @@ public void testPipelineQueryParameterIsError() throws IOException { body.endObject(); } body.endObject(); - request.withContent(body.bytes()); + request.withContent(body.bytes(), body.contentType()); } request.withParams(singletonMap("pipeline", "doesn't matter")); Exception e = expectThrows(IllegalArgumentException.class, () -> action.buildRequest(request.build())); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java index a93b017caa865..f9ab72506a84d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java @@ -186,7 +186,8 @@ public void testScrollParams() { public void testScrollEntity() throws IOException { String scroll = randomAsciiOfLength(30); HttpEntity entity = scrollEntity(scroll); - assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue()); - assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))); + assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue()); + assertThat(Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)), + containsString("\"" + scroll + "\"")); } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index 10a2da21ac948..c3b37ef7be84d 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -21,6 +21,7 @@ import io.netty.channel.Channel; import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; import org.elasticsearch.common.bytes.BytesArray; @@ -30,7 +31,12 @@ import org.elasticsearch.transport.netty4.Netty4Utils; import java.net.SocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class Netty4HttpRequest extends RestRequest { @@ -39,7 +45,7 @@ public class Netty4HttpRequest extends RestRequest { private final BytesReference content; Netty4HttpRequest(NamedXContentRegistry xContentRegistry, FullHttpRequest request, Channel channel) { - super(xContentRegistry, request.uri()); + super(xContentRegistry, request.uri(), new HttpHeadersMap(request.headers())); this.request = request; this.channel = channel; if (request.content().isReadable()) { @@ -120,14 +126,96 @@ public Channel getChannel() { return channel; } - @Override - public String header(String name) { - return request.headers().get(name); - } + /** + * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications + * and due to the underlying implementation, it performs case insensitive lookups of key to values. + * + * It is important to note that this implementation does have some downsides in that each invocation of the + * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a + * view of the underlying values. + */ + private static class HttpHeadersMap implements Map> { - @Override - public Iterable> headers() { - return request.headers().entries(); - } + private final HttpHeaders httpHeaders; + + private HttpHeadersMap(HttpHeaders httpHeaders) { + this.httpHeaders = httpHeaders; + } + + @Override + public int size() { + return httpHeaders.size(); + } + + @Override + public boolean isEmpty() { + return httpHeaders.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return key instanceof String && httpHeaders.contains((String) key); + } + + @Override + public boolean containsValue(Object value) { + return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals); + } + @Override + public List get(Object key) { + return key instanceof String ? httpHeaders.getAll((String) key) : null; + } + + @Override + public List put(String key, List value) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public List remove(Object key) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public void putAll(Map> m) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public void clear() { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public Set keySet() { + return httpHeaders.names(); + } + + @Override + public Collection> values() { + return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList()); + } + + @Override + public Set>> entrySet() { + return httpHeaders.names().stream().map(k -> new Entry>() { + + @Override + public String getKey() { + return k; + } + + @Override + public List getValue() { + return httpHeaders.getAll(k); + } + + @Override + public List setValue(List value) { + throw new UnsupportedOperationException("modifications are not supported"); + } + }).collect(Collectors.toSet()); + } + } } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java index 959a4d7c9f030..1050404563486 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java @@ -122,6 +122,7 @@ private Collection processRequestsWithBody(HttpMethod method, HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content); request.headers().add(HttpHeaderNames.HOST, "localhost"); request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()); + request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json"); requests.add(request); } return sendRequests(remoteAddress, requests); diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index c4de761c8e798..92cbb2e6e397b 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugin.mapper.MapperSizePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -107,7 +108,7 @@ public void testBasic() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true")); final String source = "{\"f\":10}"; indexRandom(true, - client().prepareIndex("test", "type", "1").setSource(source)); + client().prepareIndex("test", "type", "1").setSource(source, XContentType.JSON)); GetResponse getResponse = client().prepareGet("test", "type", "1").setStoredFields("_size").get(); assertNotNull(getResponse.getField("_size")); assertEquals(source.length(), getResponse.getField("_size").getValue()); diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index 3fa5300fb4383..2cde1b1bd07d2 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -60,7 +61,7 @@ public void testSizeEnabled() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); boolean stored = false; boolean points = false; @@ -81,7 +82,7 @@ public void testSizeDisabled() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); assertThat(doc.rootDoc().getField("_size"), nullValue()); } @@ -95,7 +96,7 @@ public void testSizeNotSet() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); assertThat(doc.rootDoc().getField("_size"), nullValue()); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java index e82ed3a18755d..b67073d7d389f 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java @@ -62,7 +62,7 @@ public void testSettingsFiltering() throws IOException { settings.toXContent(xContentBuilder, request); xContentBuilder.endObject(); String filteredSettingsString = xContentBuilder.string(); - filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build(); + filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build(); assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default")); } diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java index b5bd18aba5f14..90a580871074f 100644 --- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.backwards; import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; @@ -70,7 +71,7 @@ private void ensureGreen() throws IOException { private void createIndex(String name, Settings settings) throws IOException { assertOK(client().performRequest("PUT", name, Collections.emptyMap(), - new StringEntity("{ \"settings\": " + Strings.toString(settings) + " }"))); + new StringEntity("{ \"settings\": " + Strings.toString(settings) + " }", ContentType.APPLICATION_JSON))); } private void updateIndexSetting(String name, Settings.Builder settings) throws IOException { @@ -85,13 +86,14 @@ protected int indexDocs(String index, final int idStart, final int numDocs) thro for (int i = 0; i < numDocs; i++) { final int id = idStart + i; assertOK(client().performRequest("PUT", index + "/test/" + id, emptyMap(), - new StringEntity("{\"test\": \"test_" + id + "\"}"))); + new StringEntity("{\"test\": \"test_" + id + "\"}", ContentType.APPLICATION_JSON))); } return numDocs; } public void testSeqNoCheckpoints() throws Exception { Nodes nodes = buildNodeAndVersions(); + assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered: {}", nodes.toString()); final String bwcNames = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.joining(",")); Settings.Builder settings = Settings.builder() diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java index 2cf681e9e5a85..51dbad22dec76 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -54,7 +53,7 @@ public void testThatErrorTraceParamReturns400() throws IOException { fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); - assertThat(response.getHeader("Content-Type"), is("application/json")); + assertThat(response.getHeader("Content-Type"), is("application/json; charset=UTF-8")); assertThat(EntityUtils.toString(e.getResponse().getEntity()), is("{\"error\":\"error traces in responses are disabled.\"}")); assertThat(response.getStatusLine().getStatusCode(), is(400)); } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java index cd3cf1a38d8b2..d0b80595a26ee 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java @@ -22,7 +22,6 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; import java.util.Collections; @@ -52,7 +51,7 @@ public void testThatErrorTraceWorksByDefault() throws IOException { fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); - assertThat(response.getHeader("Content-Type"), containsString("application/json")); + assertThat(response.getHeader("Content-Type"), containsString("application/json; charset=UTF-8")); assertThat(EntityUtils.toString(response.getEntity()), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + "nested: ActionRequestValidationException[Validation Failed: 1:"))); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 1e716d4f168c3..20c869d38c6e9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MapperTestUtils; @@ -473,17 +474,23 @@ protected Engine.Index indexDoc(IndexShard shard, String type, String id) throws } protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source) throws IOException { + return indexDoc(shard, type, id, source, XContentType.JSON); + } + + protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source, XContentType xContentType) throws IOException { final Engine.Index index; if (shard.routingEntry().primary()) { index = shard.prepareIndexOnPrimary( - SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)), + SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source), + xContentType), Versions.MATCH_ANY, VersionType.INTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); } else { index = shard.prepareIndexOnReplica( - SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)), + SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source), + xContentType), randomInt(1 << 10), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); } shard.index(index); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 245ea6c08fba4..b1d66fd399b38 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -96,6 +96,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.discovery.Discovery; @@ -1226,10 +1227,10 @@ protected final IndexResponse index(String index, String type, String id, Object * return client().prepareIndex(index, type, id).setSource(source).execute().actionGet(); * *

- * where source is a String. + * where source is a JSON String. */ protected final IndexResponse index(String index, String type, String id, String source) { - return client().prepareIndex(index, type, id).setSource(source).execute().actionGet(); + return client().prepareIndex(index, type, id).setSource(source, XContentType.JSON).execute().actionGet(); } /** @@ -1387,7 +1388,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet()); String index = RandomPicks.randomFrom(random, indices); bogusIds.add(new Tuple<>(index, id)); - builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}")); + builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON)); } } final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index c7203f971923f..633a04f5f60c2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -294,7 +294,7 @@ public final void after() throws Exception { } private void ensureNoWarnings() throws IOException { - //Check that there are no unaccounted warning headers. These should be checked with {@link #checkWarningHeaders(String...)} in the + //Check that there are no unaccounted warning headers. These should be checked with {@link #assertWarnings(String...)} in the //appropriate test try { final List warnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER); diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index 31380e25eb3d4..dcad7187fbe68 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -41,7 +41,7 @@ public static Map convertToMap(ToXContent part) throws IOExcepti builder.startObject(); part.toXContent(builder, EMPTY_PARAMS); builder.endObject(); - return XContentHelper.convertToMap(builder.bytes(), false).v2(); + return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java index 3d1ce29143208..bca4af7bd6036 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java @@ -19,9 +19,9 @@ package org.elasticsearch.test.rest; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -53,8 +53,8 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - return super.newBuilder(autoDetectSource, useFiltering); + public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + return super.newBuilder(requestContentType, useFiltering); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java index ae8c4c82c4663..83caf0293e0ab 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java @@ -21,28 +21,31 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest; +import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class FakeRestRequest extends RestRequest { - private final Map headers; private final BytesReference content; private final Method method; - + private final SocketAddress remoteAddress; public FakeRestRequest() { - this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/"); + this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/", null); } - private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map headers, Map params, - BytesReference content, Method method, String path) { - super(xContentRegistry, params, path); - this.headers = headers; + private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map> headers, Map params, + BytesReference content, Method method, String path, SocketAddress remoteAddress) { + super(xContentRegistry, params, path, headers); this.content = content; this.method = method; + this.remoteAddress = remoteAddress; } @Override @@ -66,19 +69,14 @@ public BytesReference content() { } @Override - public String header(String name) { - return headers.get(name); - } - - @Override - public Iterable> headers() { - return headers.entrySet(); + public SocketAddress getRemoteAddress() { + return remoteAddress; } public static class Builder { private final NamedXContentRegistry xContentRegistry; - private Map headers = new HashMap<>(); + private Map> headers = new HashMap<>(); private Map params = new HashMap<>(); @@ -88,11 +86,13 @@ public static class Builder { private Method method = Method.GET; + private SocketAddress address = null; + public Builder(NamedXContentRegistry xContentRegistry) { this.xContentRegistry = xContentRegistry; } - public Builder withHeaders(Map headers) { + public Builder withHeaders(Map> headers) { this.headers = headers; return this; } @@ -102,8 +102,11 @@ public Builder withParams(Map params) { return this; } - public Builder withContent(BytesReference content) { + public Builder withContent(BytesReference content, XContentType xContentType) { this.content = content; + if (xContentType != null) { + headers.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + } return this; } @@ -117,8 +120,13 @@ public Builder withMethod(Method method) { return this; } + public Builder withRemoteAddress(SocketAddress address) { + this.address = address; + return this; + } + public FakeRestRequest build() { - return new FakeRestRequest(xContentRegistry, headers, params, content, method, path); + return new FakeRestRequest(xContentRegistry, headers, params, content, method, path, address); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index c18dad907bd6e..aa51efaa33319 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -55,13 +55,15 @@ public class ClientYamlTestClient { private final ClientYamlSuiteRestSpec restSpec; private final RestClient restClient; private final Version esVersion; + private final Map hostVersionMap; public ClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, - Version esVersion) throws IOException { + Version esVersion, Map hostVersionMap) throws IOException { assert hosts.size() > 0; this.restSpec = restSpec; this.restClient = restClient; this.esVersion = esVersion; + this.hostVersionMap = hostVersionMap; } public Version getEsVersion() { @@ -86,9 +88,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params // And everything else is a url parameter! try { Response response = restClient.performRequest(method, path, queryStringParams, entity); - return new ClientYamlTestResponse(response); + return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost())); } catch(ResponseException e) { - throw new ClientYamlTestResponseException(e); + throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost())); } } @@ -122,6 +124,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params if (supportedMethods.contains("GET") && RandomizedTest.rarely()) { logger.debug("sending the request body as source param with GET method"); queryStringParams.put("source", body); + if (esVersion.after(Version.V_5_3_0_UNRELEASED)) { // TODO make onOrAfter with backport + queryStringParams.put("source_content_type", ContentType.APPLICATION_JSON.toString()); + } requestMethod = "GET"; } else { requestMethod = RandomizedTest.randomFrom(supportedMethods); @@ -168,9 +173,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params logger.debug("calling api [{}]", apiName); try { Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders); - return new ClientYamlTestResponse(response); + return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost())); } catch(ResponseException e) { - throw new ClientYamlTestResponseException(e); + throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost())); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 8215105887ded..27653a13f2940 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -71,9 +71,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params throw e; } finally { // if we hit a bad exception the response is null - Object repsponseBody = response != null ? response.getBody() : null; + Object responseBody = response != null ? response.getBody() : null; //we always stash the last response body - stash.stashValue("body", repsponseBody); + stash.stashValue("body", responseBody); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java index 481ae752d05f0..f47cfd68d4e19 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java @@ -21,6 +21,7 @@ import org.apache.http.Header; import org.apache.http.client.methods.HttpHead; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.XContentType; @@ -37,10 +38,12 @@ public class ClientYamlTestResponse { private final Response response; private final String body; + private final Version nodeVersion; private ObjectPath parsedResponse; - ClientYamlTestResponse(Response response) throws IOException { + ClientYamlTestResponse(Response response, Version version) throws IOException { this.response = response; + this.nodeVersion = version; if (response.getEntity() != null) { try { this.body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); @@ -80,7 +83,18 @@ public List getWarningHeaders() { List warningHeaders = new ArrayList<>(); for (Header header : response.getHeaders()) { if (header.getName().equals("Warning")) { - warningHeaders.add(header.getValue()); + if (nodeVersion.after(Version.V_5_3_0_UNRELEASED) && response.getRequestLine().getMethod().equals("GET") + && response.getRequestLine().getUri().contains("source") + && response.getRequestLine().getUri().contains("source_content_type") == false && header.getValue().startsWith( + "Deprecated use of the [source] parameter without the [source_content_type] parameter.")) { + // this is because we do not send the source content type header when the node is 5.3.0 or below and the request + // might have been sent to a node with a version > 5.3.0 when running backwards 5.0 tests. The Java RestClient + // has control of the node the request is sent to so we can only detect this after the fact right now + // TODO remove this when we bump versions + } else { + warningHeaders.add(header.getValue()); + } + } } return warningHeaders; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java index 7d983d480296b..48a7ee578d51a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.rest.yaml; +import org.elasticsearch.Version; import org.elasticsearch.client.ResponseException; import java.io.IOException; @@ -32,10 +33,10 @@ public class ClientYamlTestResponseException extends IOException { private final ClientYamlTestResponse restTestResponse; private final ResponseException responseException; - ClientYamlTestResponseException(ResponseException responseException) throws IOException { + ClientYamlTestResponseException(ResponseException responseException, Version version) throws IOException { super(responseException); this.responseException = responseException; - this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse()); + this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse(), version); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index a45d92c9699de..411f657329d8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -53,6 +53,7 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -125,6 +126,7 @@ public void initAndResetContext() throws IOException { validateSpec(restSpec); List hosts = getClusterHosts(); RestClient restClient = client(); + Tuple> versionMapTuple = readVersionsFromInfo(restClient, hosts.size()); Version esVersion; try { Tuple versionVersionTuple = readVersionsFromCatNodes(restClient); @@ -135,13 +137,14 @@ public void initAndResetContext() throws IOException { } catch (ResponseException ex) { if (ex.getResponse().getStatusLine().getStatusCode() == 403) { logger.warn("Fallback to simple info '/' request, _cat/nodes is not authorized"); - esVersion = readVersionsFromInfo(restClient, hosts.size()); + esVersion = versionMapTuple.v1(); logger.info("initializing yaml client, minimum es version: [{}] hosts: {}", esVersion, hosts); } else { throw ex; } } - ClientYamlTestClient clientYamlTestClient = new ClientYamlTestClient(restSpec, restClient, hosts, esVersion); + ClientYamlTestClient clientYamlTestClient = + new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, versionMapTuple.v2()); restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient); adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient); String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); @@ -290,7 +293,7 @@ public static void clearStatic() { private static Tuple readVersionsFromCatNodes(RestClient restClient) throws IOException { // we simply go to the _cat/nodes API and parse all versions in the cluster Response response = restClient.performRequest("GET", "/_cat/nodes", Collections.singletonMap("h", "version,master")); - ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); + ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT); String nodesCatResponse = restTestResponse.getBodyAsString(); String[] split = nodesCatResponse.split("\n"); Version version = null; @@ -313,12 +316,13 @@ private static Tuple readVersionsFromCatNodes(RestClient restC return new Tuple<>(version, masterVersion); } - private static Version readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException { + private static Tuple> readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException { Version version = null; + Map hostVersionMap = new HashMap<>(); for (int i = 0; i < numHosts; i++) { //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster Response response = restClient.performRequest("GET", "/"); - ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); + ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT); Object latestVersion = restTestResponse.evaluate("version.number"); if (latestVersion == null) { throw new RuntimeException("elasticsearch version not found in the response"); @@ -329,8 +333,9 @@ private static Version readVersionsFromInfo(RestClient restClient, int numHosts) } else if (version.onOrAfter(currentVersion)) { version = currentVersion; } + hostVersionMap.put(response.getHost(), currentVersion); } - return version; + return new Tuple<>(version, Collections.unmodifiableMap(hostVersionMap)); } public void test() throws IOException {