From e4b2d0865f1ee659f77ff1c54691c8f28ed9ee99 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 12 Jan 2017 14:32:31 -0500 Subject: [PATCH 01/34] Require a valid content type for all rest requests with content This change enforces that all incoming rest requests have a valid and supported content type header before the request is dispatched. The content type header is parsed to the matching XContentType value with the only exception being for plain text requests. This value is then passed on with the content bytes so that we can reduce the number of places where we need to autodetect the content type. As part of this, many transport requests and builders were updated to provide methods that accepted the XContentType along with the bytes and the methods that would rely on autodetection have been deprecated. Closes #19388 --- .../transport/TransportClientBenchmark.java | 3 +- .../action/bulk/NoopBulkRequestBuilder.java | 11 +- .../noop/action/bulk/RestNoopBulkAction.java | 3 +- .../storedscripts/PutStoredScriptRequest.java | 31 ++- .../PutStoredScriptRequestBuilder.java | 6 + .../indices/create/CreateIndexRequest.java | 37 ++++ .../create/CreateIndexRequestBuilder.java | 41 ++++ .../mapping/put/PutMappingRequest.java | 30 ++- .../mapping/put/PutMappingRequestBuilder.java | 11 + .../template/put/PutIndexTemplateRequest.java | 96 +++++++- .../put/PutIndexTemplateRequestBuilder.java | 55 +++++ .../action/bulk/BulkRequest.java | 59 ++++- .../action/bulk/BulkRequestBuilder.java | 22 ++ .../action/bulk/TransportShardBulkAction.java | 2 +- .../action/index/IndexRequest.java | 80 ++++++- .../action/index/IndexRequestBuilder.java | 51 ++++- .../action/ingest/PutPipelineRequest.java | 34 +++ .../ingest/PutPipelineRequestBuilder.java | 6 + .../ingest/SimulatePipelineRequest.java | 29 +++ .../SimulatePipelineRequestBuilder.java | 7 + .../SimulatePipelineTransportAction.java | 2 +- .../action/update/TransportUpdateAction.java | 3 +- .../action/update/UpdateRequest.java | 64 +++++- .../action/update/UpdateRequestBuilder.java | 60 +++++ .../client/ClusterAdminClient.java | 14 ++ .../client/support/AbstractClient.java | 11 + .../common/settings/Settings.java | 16 ++ .../loader/SettingsLoaderFactory.java | 23 +- .../common/xcontent/XContentBuilder.java | 15 ++ .../common/xcontent/XContentGenerator.java | 6 + .../common/xcontent/XContentHelper.java | 31 ++- .../common/xcontent/XContentType.java | 20 +- .../xcontent/json/JsonXContentGenerator.java | 15 ++ .../index/mapper/DocumentMapper.java | 4 +- .../index/mapper/SourceToParse.java | 20 +- .../ingest/PipelineConfiguration.java | 42 +++- .../elasticsearch/ingest/PipelineStore.java | 4 +- .../rest/AbstractRestChannel.java | 19 +- .../org/elasticsearch/rest/RestChannel.java | 3 +- .../elasticsearch/rest/RestController.java | 59 +++-- .../org/elasticsearch/rest/RestHandler.java | 1 - .../org/elasticsearch/rest/RestRequest.java | 206 +++++++++++++++++- .../cluster/RestPutStoredScriptAction.java | 2 +- .../admin/indices/RestCreateIndexAction.java | 2 +- .../indices/RestPutIndexTemplateAction.java | 2 +- .../admin/indices/RestPutMappingAction.java | 5 +- .../indices/RestUpdateSettingsAction.java | 4 +- .../rest/action/document/RestBulkAction.java | 2 +- .../action/document/RestGetSourceAction.java | 2 +- .../rest/action/document/RestIndexAction.java | 2 +- .../action/ingest/RestPutPipelineAction.java | 6 +- .../ingest/RestSimulatePipelineAction.java | 6 +- .../action/search/RestClearScrollAction.java | 25 ++- .../action/search/RestMultiSearchAction.java | 12 +- .../action/search/RestSearchScrollAction.java | 22 +- .../elasticsearch/script/ScriptMetaData.java | 9 +- .../elasticsearch/script/ScriptService.java | 9 +- .../admin/cluster/node/tasks/TasksIT.java | 6 +- .../reroute/ClusterRerouteRequestTests.java | 2 +- .../PutStoredScriptRequestTests.java | 60 +++++ .../CreateIndexRequestBuilderTests.java | 5 +- .../admin/indices/create/ShrinkIndexIT.java | 19 +- .../mapping/put/PutMappingRequestTests.java | 49 ++++- .../indices/template/BWCTemplateTests.java | 11 +- .../put/PutIndexTemplateRequestTests.java | 37 ++++ .../action/bulk/BulkIntegrationIT.java | 3 +- .../bulk/BulkProcessorClusterSettingsIT.java | 7 +- .../action/bulk/BulkRequestModifierTests.java | 3 +- .../action/bulk/BulkRequestTests.java | 27 +-- .../action/bulk/BulkWithUpdatesIT.java | 35 +-- .../bulk/TransportBulkActionTookTests.java | 3 +- .../index/IndexRequestBuilderTests.java | 5 +- .../action/index/IndexRequestTests.java | 40 +++- .../ingest/PutPipelineRequestTests.java | 62 ++++++ .../ingest/SimulatePipelineRequestTests.java | 35 ++- .../search/MultiSearchRequestTests.java | 4 +- .../support/WaitActiveShardCountIT.java | 11 +- .../action/update/UpdateRequestTests.java | 5 +- .../elasticsearch/aliases/IndexAliasesIT.java | 87 ++++---- .../client/AbstractClientHeadersTestCase.java | 3 +- .../cluster/SimpleDataNodesIT.java | 10 +- .../DiscoveryWithServiceDisruptionsIT.java | 6 +- .../document/DocumentActionsIT.java | 3 +- .../elasticsearch/document/ShardInfoIT.java | 11 +- .../org/elasticsearch/get/GetActionIT.java | 19 +- .../index/IndexRequestBuilderIT.java | 10 +- .../index/IndexServiceTests.java | 7 +- .../index/mapper/MapperServiceTests.java | 3 +- .../mapper/UpdateMappingOnClusterIT.java | 4 - .../ESIndexLevelReplicationTestCase.java | 5 +- .../IndexLevelReplicationTests.java | 3 +- .../index/shard/IndexShardIT.java | 32 +-- ...DateMathIndexExpressionsIntegrationIT.java | 13 +- .../IndexingMemoryControllerTests.java | 3 +- .../elasticsearch/indices/flush/FlushIT.java | 5 +- .../flush/SyncedFlushSingleNodeTests.java | 15 +- .../breaker/CircuitBreakerServiceIT.java | 3 +- .../indices/recovery/IndexRecoveryIT.java | 5 +- .../indices/stats/IndexStatsIT.java | 11 +- .../template/SimpleIndexTemplateIT.java | 25 ++- .../elasticsearch/ingest/IngestClientIT.java | 13 +- .../ingest/IngestMetadataTests.java | 37 ++-- ...gestProcessorNotInstalledOnAllNodesIT.java | 7 +- .../ingest/PipelineConfigurationTests.java | 80 +++++++ .../ingest/PipelineExecutionServiceTests.java | 11 +- .../ingest/PipelineStoreTests.java | 32 +-- .../org/elasticsearch/mget/SimpleMgetIT.java | 3 +- .../elasticsearch/recovery/RelocationIT.java | 7 +- .../recovery/SimpleRecoveryIT.java | 5 +- .../rest/BytesRestResponseTests.java | 13 +- .../rest/RestControllerTests.java | 68 ++++-- .../elasticsearch/rest/RestRequestTests.java | 81 +++++-- .../admin/indices/RestAnalyzeActionTests.java | 4 +- .../rest/action/cat/RestTableTests.java | 16 +- .../script/ScriptMetaDataTests.java | 29 +-- .../script/ScriptServiceTests.java | 11 +- .../aggregations/bucket/ChildrenIT.java | 7 +- .../search/aggregations/bucket/NestedIT.java | 5 +- .../bucket/TermsShardMinDocCountIT.java | 7 +- .../search/child/ChildQuerySearchIT.java | 29 +-- .../search/child/ParentFieldLoadingIT.java | 19 +- .../search/fetch/subphase/InnerHitsIT.java | 46 ++-- .../search/fields/SearchFieldsIT.java | 5 +- .../elasticsearch/search/geo/GeoFilterIT.java | 11 +- .../search/geo/GeoShapeQueryTests.java | 8 +- .../search/msearch/MultiSearchIT.java | 3 +- .../search/query/QueryStringIT.java | 7 +- .../search/query/SearchQueryIT.java | 3 +- .../search/query/SimpleQueryStringIT.java | 3 +- .../scroll/RestClearScrollActionTests.java | 4 +- .../scroll/RestSearchScrollActionTests.java | 4 +- .../search/simple/SimpleSearchIT.java | 23 +- .../search/sort/FieldSortIT.java | 9 +- .../DedicatedClusterSnapshotRestoreIT.java | 2 - .../SharedClusterSnapshotRestoreIT.java | 3 +- .../org/elasticsearch/update/UpdateIT.java | 3 +- .../elasticsearch/update/UpdateNoopIT.java | 2 +- .../test/rest/CreatedLocationHeaderIT.java | 9 +- .../expression/IndexedExpressionTests.java | 3 +- .../MultiSearchTemplateRequestTests.java | 4 +- .../script/mustache/SearchTemplateIT.java | 31 +-- .../percolator/PercolatorQuerySearchIT.java | 5 +- .../reindex/remote/RemoteRequestBuilders.java | 18 +- .../remote/RemoteScrollableHitSource.java | 3 +- .../index/reindex/CancelTests.java | 5 +- .../index/reindex/RestReindexActionTests.java | 2 +- .../remote/RemoteRequestBuildersTests.java | 5 +- .../http/netty4/Netty4HttpRequest.java | 106 ++++++++- .../http/netty4/Netty4HttpClient.java | 1 + .../index/mapper/size/SizeMappingIT.java | 3 +- qa/backwards-5.0/build.gradle | 2 +- .../elasticsearch/backwards/IndexingIT.java | 6 +- .../http/DetailedErrorsDisabledIT.java | 3 +- .../http/DetailedErrorsEnabledIT.java | 3 +- .../elasticsearch/test/ESIntegTestCase.java | 7 +- .../test/rest/FakeRestChannel.java | 6 +- .../test/rest/FakeRestRequest.java | 44 ++-- .../test/rest/yaml/ClientYamlTestClient.java | 1 + 158 files changed, 2329 insertions(+), 632 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java create mode 100644 core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java create mode 100644 core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java index 6d6e5ade8275a..c5a544eb7854a 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugin.noop.NoopPlugin; import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction; @@ -80,7 +81,7 @@ public TransportBulkRequestExecutor(TransportClient client, String indexName, St public boolean bulkIndex(List bulkData) { NoopBulkRequestBuilder builder = NoopBulkAction.INSTANCE.newRequestBuilder(client); for (String bulkItem : bulkData) { - builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8))); + builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } BulkResponse bulkResponse; try { diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java index ceaf9f8cc9d17..1034e722e8789 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java @@ -33,6 +33,7 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; public class NoopBulkRequestBuilder extends ActionRequestBuilder implements WriteRequestBuilder { @@ -95,17 +96,17 @@ public NoopBulkRequestBuilder add(UpdateRequestBuilder request) { /** * Adds a framed data in binary format */ - public NoopBulkRequestBuilder add(byte[] data, int from, int length) throws Exception { - request.add(data, from, length, null, null); + public NoopBulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception { + request.add(data, from, length, null, null, xContentType); return this; } /** * Adds a framed data in binary format */ - public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) - throws Exception { - request.add(data, from, length, defaultIndex, defaultType); + public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws Exception { + request.add(data, from, length, defaultIndex, defaultType, xContentType); return this; } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java index c3f3205fc2c22..8bcc4ea82ce60 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -75,7 +75,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); - bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true); + bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true, + request.getXContentType()); // short circuit the call to the transport layer return channel -> { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index cfe153d7d9641..079dfb810242b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -19,12 +19,14 @@ package org.elasticsearch.action.admin.cluster.storedscripts; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -35,6 +37,7 @@ public class PutStoredScriptRequest extends AcknowledgedRequest { @@ -40,9 +41,14 @@ public PutStoredScriptRequestBuilder setId(String id) { return this; } + @Deprecated public PutStoredScriptRequestBuilder setSource(BytesReference source) { request.script(source); return this; } + public PutStoredScriptRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.script(source, xContentType); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 203483d89b3f4..a6cc3e9ced8b8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -331,11 +331,20 @@ public CreateIndexRequest alias(Alias alias) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public CreateIndexRequest source(String source) { return source(source.getBytes(StandardCharsets.UTF_8)); } + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(String source, XContentType xContentType) { + return source(source.getBytes(StandardCharsets.UTF_8), xContentType); + } + /** * Sets the settings and mappings as a single source. */ @@ -345,7 +354,9 @@ public CreateIndexRequest source(XContentBuilder source) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], XContentType)} */ + @Deprecated public CreateIndexRequest source(byte[] source) { return source(source, 0, source.length); } @@ -353,6 +364,15 @@ public CreateIndexRequest source(byte[] source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -360,8 +380,25 @@ public CreateIndexRequest source(byte[] source, int offset, int length) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequest source(BytesReference source) { XContentType xContentType = XContentFactory.xContentType(source); + source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(BytesReference source, XContentType xContentType) { if (xContentType != null) { source(XContentHelper.convertToMap(source, false).v2()); } else { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index eaae4d53b73fd..c9ac84c0b62f7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -191,7 +192,9 @@ public CreateIndexRequestBuilder addAlias(Alias alias) { /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public CreateIndexRequestBuilder setSource(String source) { request.source(source); return this; @@ -200,6 +203,16 @@ public CreateIndexRequestBuilder setSource(String source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; @@ -208,6 +221,16 @@ public CreateIndexRequestBuilder setSource(BytesReference source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source) { request.source(source); return this; @@ -216,11 +239,29 @@ public CreateIndexRequestBuilder setSource(byte[] source) { /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; } + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + /** * Sets the settings and mappings as a single source. */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 152bc516549b9..025a47c47fcf6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -22,19 +22,24 @@ import com.carrotsearch.hppc.ObjectHashSet; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -245,7 +250,7 @@ public static XContentBuilder buildFromSimplifiedDef(String type, Object... sour */ public PutMappingRequest source(XContentBuilder mappingBuilder) { try { - return source(mappingBuilder.string()); + return source(mappingBuilder.string(), mappingBuilder.contentType()); } catch (IOException e) { throw new IllegalArgumentException("Failed to build json for mapping request", e); } @@ -259,7 +264,7 @@ public PutMappingRequest source(Map mappingSource) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(mappingSource); - return source(builder.string()); + return source(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + mappingSource + "]", e); } @@ -267,9 +272,22 @@ public PutMappingRequest source(Map mappingSource) { /** * The mapping source definition. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public PutMappingRequest source(String mappingSource) { - this.source = mappingSource; + return source(mappingSource, XContentFactory.xContentType(mappingSource)); + } + + /** + * The mapping source definition. + */ + public PutMappingRequest source(String mappingSource, XContentType xContentType) { + try { + this.source = XContentHelper.convertToJson(new BytesArray(mappingSource.getBytes(StandardCharsets.UTF_8)), false, xContentType); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert mapping source to json", e); + } return this; } @@ -290,7 +308,11 @@ public void readFrom(StreamInput in) throws IOException { indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); - source = in.readString(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + source = in.readString(); + } else { + source = XContentHelper.convertToJson(new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)), false); + } updateAllTypes = in.readBoolean(); readTimeout(in); concreteIndex = in.readOptionalWriteable(Index::new); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index c21c40cf041ea..012a593ebc473 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import java.util.Map; @@ -82,12 +83,22 @@ public PutMappingRequestBuilder setSource(Map mappingSource) { /** * The mapping source definition. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public PutMappingRequestBuilder setSource(String mappingSource) { request.source(mappingSource); return this; } + /** + * The mapping source definition. + */ + public PutMappingRequestBuilder setSource(String mappingSource, XContentType xContentType) { + request.source(mappingSource, xContentType); + return this; + } + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index a7d6241d31e2b..4af876f6293b7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -179,13 +179,23 @@ public PutIndexTemplateRequest settings(Settings.Builder settings) { } /** - * The settings to create the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml format). + * @deprecated use {@link #settings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * The settings to create the index template with (either json/yaml format). + */ + public PutIndexTemplateRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * The settings to crete the index template with (either json/yaml/properties format). */ @@ -193,7 +203,7 @@ public PutIndexTemplateRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -209,9 +219,35 @@ public Settings settings() { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest mapping(String type, String source) { - mappings.put(type, source); + XContentType xContentType = XContentFactory.xContentType(source); + return mapping(type, source, xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { + if (xContentType == null) { + throw new IllegalArgumentException("could not determine xcontent type"); + } else if (xContentType == XContentType.JSON) { + mappings.put(type, source); + } else { + try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, source); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + builder.copyCurrentStructure(parser); + mappings.put(type, builder.string()); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } return this; } @@ -235,11 +271,10 @@ public String cause() { */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { try { - mappings.put(type, source.string()); + return mapping(type, source.string(), source.contentType()); } catch (IOException e) { throw new IllegalArgumentException("Failed to build json for mapping request", e); } - return this; } /** @@ -256,7 +291,7 @@ public PutIndexTemplateRequest mapping(String type, Map source) try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -280,7 +315,7 @@ public Map mappings() { */ public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { try { - return source(templateBuilder.bytes()); + return source(templateBuilder.bytes(), templateBuilder.contentType()); } catch (Exception e) { throw new IllegalArgumentException("Failed to build json for template request", e); } @@ -350,7 +385,9 @@ public PutIndexTemplateRequest source(Map templateSource) { /** * The template source definition. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest source(String templateSource) { return source(XContentHelper.convertToMap(XContentFactory.xContent(templateSource), templateSource, true)); } @@ -358,6 +395,15 @@ public PutIndexTemplateRequest source(String templateSource) { /** * The template source definition. */ + public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source) { return source(source, 0, source.length); } @@ -365,6 +411,15 @@ public PutIndexTemplateRequest source(byte[] source) { /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -372,10 +427,26 @@ public PutIndexTemplateRequest source(byte[] source, int offset, int length) { /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(BytesReference source) { return source(XContentHelper.convertToMap(source, true).v2()); } + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2()); + } + public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { customs.put(custom.type(), custom); return this; @@ -470,8 +541,15 @@ public void readFrom(StreamInput in) throws IOException { create = in.readBoolean(); settings = readSettingsFromStream(in); int size = in.readVInt(); - for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + for (int i = 0; i < size; i++) { + mappings.put(in.readString(), in.readString()); + } + } else { + // we cannot expect that the string is json from older versions so we may need to convert + for (int i = 0; i < size; i++) { + mapping(in.readString(), in.readString()); + } } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index c1db96ae7ce5c..60f08a13ef0b9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Collections; import java.util.List; @@ -101,12 +102,22 @@ public PutIndexTemplateRequestBuilder setSettings(Settings.Builder settings) { /** * The settings to crete the index template with (either json/yaml/properties format) + * @deprecated use {@link #setSettings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * The settings to crete the index template with (either json/yaml/properties format) + */ + public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * The settings to crete the index template with (either json/yaml/properties format) */ @@ -120,12 +131,26 @@ public PutIndexTemplateRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } + /** + * Adds mapping that will be added when the index template gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type/format of the source + */ + public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). @@ -226,12 +251,22 @@ public PutIndexTemplateRequestBuilder setSource(Map templateSource) { /** * The template source definition. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(String templateSource) { request.source(templateSource); return this; } + /** + * The template source definition. + */ + public PutIndexTemplateRequestBuilder setSource(String templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + /** * The template source definition. */ @@ -242,7 +277,9 @@ public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) { /** * The template source definition. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) { request.source(templateSource); return this; @@ -251,8 +288,26 @@ public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) { /** * The template source definition. */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + + /** + * The template source definition. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length) { request.source(templateSource, offset, length); return this; } + + /** + * The template source definition. + */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length, XContentType xContentType) { + request.source(templateSource, offset, length, xContentType); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 20d5e64f49a7f..371659586f9af 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -244,7 +245,9 @@ public long estimatedSizeInBytes() { /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequest add(byte[] data, int from, int length) throws IOException { return add(data, from, length, null, null); } @@ -252,6 +255,15 @@ public BulkRequest add(byte[] data, int from, int length) throws IOException { /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException { + return add(data, from, length, null, null, xContentType); + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(new BytesArray(data, from, length), defaultIndex, defaultType); } @@ -259,6 +271,17 @@ public BulkRequest add(byte[] data, int from, int length, @Nullable String defau /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(new BytesArray(data, from, length), defaultIndex, defaultType, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, true); } @@ -266,12 +289,40 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null /** * Adds a framed data in binary format */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, true, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, boolean, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex); } + /** + * Adds a framed data in binary format + */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex, xContentType); + } + + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws IOException { - XContent xContent = XContentFactory.xContent(data); + XContentType xContentType = XContentFactory.xContentType(data); + return add(data, defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, payload, + allowExplicitIndex, xContentType); + } + + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String + defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String + defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex, XContentType xContentType) throws IOException { + XContent xContent = xContentType.xContent(); int line = 0; int from = 0; int length = data.length(); @@ -387,16 +438,16 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null if ("index".equals(action)) { if (opType == null) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) - .setPipeline(pipeline).source(data.slice(from, nextMarker - from)), payload); + .setPipeline(pipeline).source(data.slice(from, nextMarker - from), xContentType), payload); } else { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create("create".equals(opType)).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } } else if ("create".equals(action)) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create(true).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } else if ("update".equals(action)) { UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict) .version(version).versionType(versionType) diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index c48a8f507b862..8f634fa28a41b 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes @@ -97,7 +98,9 @@ public BulkRequestBuilder add(UpdateRequestBuilder request) { /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length) throws Exception { request.add(data, from, length, null, null); return this; @@ -106,11 +109,30 @@ public BulkRequestBuilder add(byte[] data, int from, int length) throws Exceptio /** * Adds a framed data in binary format */ + public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception { + request.add(data, from, length, null, null, xContentType); + return this; + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception { request.add(data, from, length, defaultIndex, defaultType); return this; } + /** + * Adds a framed data in binary format + */ + public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws Exception { + request.add(data, from, length, defaultIndex, defaultType, xContentType); + return this; + } + /** * Sets the number of shard copies that must be active before proceeding with the write. * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 2a9ee444941b9..5df7d550d2fce 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -327,7 +327,7 @@ private UpdateResultHolder executeUpdateRequest(UpdateRequest updateRequest, Ind if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) || (updateRequest.fields() != null && updateRequest.fields().length > 0)) { Tuple> sourceAndContent = - XContentHelper.convertToMap(indexSourceAsBytes, true); + XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType()); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index d1a9927c67c5e..c6bb1c353802b 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -55,10 +55,10 @@ * created using {@link org.elasticsearch.client.Requests#indexRequest(String)}. * * The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and - * {@link #source(byte[])} to be set. + * {@link #source(byte[], XContentType)} to be set. * - * The source (content to index) can be set in its bytes form using ({@link #source(byte[])}), - * its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} + * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}), + * its string form ({@link #source(String, XContentType)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} * ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}). * * If the {@link #id(String)} is not set, it will be automatically generated. @@ -103,7 +103,7 @@ public IndexRequest() { /** * Constructs a new index request against the specific index. The {@link #type(String)} - * {@link #source(byte[])} must be set. + * {@link #source(byte[], XContentType)} must be set. */ public IndexRequest(String index) { this.index = index; @@ -179,14 +179,16 @@ public ActionRequestValidationException validate() { } /** - * The content type that will be used when generating a document from user provided objects like Maps. + * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the + * source at index time */ public XContentType getContentType() { return contentType; } /** - * Sets the content type that will be used when generating a document from user provided objects (like Map). + * Sets the content type. This will be used when generating a document from user provided objects (like Map) and when parsing the + * source at index time */ public IndexRequest contentType(XContentType contentType) { this.contentType = contentType; @@ -284,7 +286,7 @@ public BytesReference source() { } public Map sourceAsMap() { - return XContentHelper.convertToMap(source, false).v2(); + return XContentHelper.convertToMap(source, false, contentType).v2(); } /** @@ -314,11 +316,24 @@ public IndexRequest source(Map source, XContentType contentType) throws Elastics /** * Sets the document source to index. * - * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #source(byte[])}. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public IndexRequest source(String source) { this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); + this.contentType = XContentFactory.xContentType(source); + return this; + } + + /** + * Sets the document source to index. + * + * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #source(byte[], XContentType)}. + */ + public IndexRequest source(String source, XContentType xContentType) { + this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); + this.contentType = xContentType; return this; } @@ -327,6 +342,7 @@ public IndexRequest source(String source) { */ public IndexRequest source(XContentBuilder sourceBuilder) { source = sourceBuilder.bytes(); + contentType = sourceBuilder.contentType(); return this; } @@ -360,19 +376,40 @@ public IndexRequest source(Object... source) { /** * Sets the document to index in bytes form. + * @deprecated use {@link #source(BytesReference, XContentType)} */ + @Deprecated public IndexRequest source(BytesReference source) { this.source = source; + this.contentType = XContentFactory.xContentType(source); return this; } /** * Sets the document to index in bytes form. */ + public IndexRequest source(BytesReference source, XContentType xContentType) { + this.source = source; + this.contentType = xContentType; + return this; + } + + /** + * Sets the document to index in bytes form. + * @deprecated use {@link #source(byte[], XContentType)} + */ + @Deprecated public IndexRequest source(byte[] source) { return source(source, 0, source.length); } + /** + * Sets the document to index in bytes form. + */ + public IndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). @@ -380,9 +417,26 @@ public IndexRequest source(byte[] source) { * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #source(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequest source(byte[] source, int offset, int length) { this.source = new BytesArray(source, offset, length); + this.contentType = XContentFactory.xContentType(source); + return this; + } + + /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + */ + public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + this.source = new BytesArray(source, offset, length); + this.contentType = xContentType; return this; } @@ -515,6 +569,11 @@ public void readFrom(StreamInput in) throws IOException { pipeline = in.readOptionalString(); isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + contentType = XContentType.readFrom(in); + } else { + contentType = XContentFactory.xContentType(source); + } } @Override @@ -543,6 +602,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(pipeline); out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + contentType.writeTo(out); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index f7df8bffced3d..7e249aba23278 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -82,12 +82,22 @@ public IndexRequestBuilder setParent(String parent) { /** * Sets the source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; } + /** + * Sets the source. + */ + public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Index the Map as a JSON. * @@ -112,13 +122,26 @@ public IndexRequestBuilder setSource(Map source, XContentType content * Sets the document source to index. *

* Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #setSource(byte[])}. + * or using the {@link #setSource(byte[], XContentType)}. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(String source) { request.source(source); return this; } + /** + * Sets the document source to index. + *

+ * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #setSource(byte[], XContentType)}. + */ + public IndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Sets the content source to index. */ @@ -129,12 +152,22 @@ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) { /** * Sets the document to index in bytes form. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source) { request.source(source); return this; } + /** + * Sets the document to index in bytes form. + */ + public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). @@ -142,12 +175,28 @@ public IndexRequestBuilder setSource(byte[] source) { * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; } + /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + * @param xContentType The type/format of the source + */ + public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + /** * Constructs a simple document with a field name and value pairs. *

diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 10416146ba853..03bce969026d5 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Objects; @@ -34,7 +37,9 @@ public class PutPipelineRequest extends AcknowledgedRequest private String id; private BytesReference source; + private XContentType xContentType = XContentType.JSON; + @Deprecated public PutPipelineRequest(String id, BytesReference source) { if (id == null) { throw new IllegalArgumentException("id is missing"); @@ -45,6 +50,23 @@ public PutPipelineRequest(String id, BytesReference source) { this.id = id; this.source = source; + this.xContentType = XContentFactory.xContentType(source); + } + + public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { + if (id == null) { + throw new IllegalArgumentException("id is missing"); + } + if (source == null) { + throw new IllegalArgumentException("source is missing"); + } + if (xContentType == null) { + throw new IllegalArgumentException("unable to determine source type"); + } + + this.id = id; + this.source = source; + this.xContentType = xContentType; } PutPipelineRequest() { @@ -63,11 +85,20 @@ public BytesReference getSource() { return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); source = in.readBytesReference(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -75,5 +106,8 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java index bd927115fb5ff..c03b3b84f8b5b 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class PutPipelineRequestBuilder extends ActionRequestBuilder { @@ -29,8 +30,13 @@ public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction a super(client, action, new PutPipelineRequest()); } + @Deprecated public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source) { super(client, action, new PutPipelineRequest(id, source)); } + public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source, + XContentType xContentType) { + super(client, action, new PutPipelineRequest(id, source, xContentType)); + } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index ef7b5e3d5bbed..7dbac307fc6dd 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Pipeline; @@ -42,12 +45,26 @@ public class SimulatePipelineRequest extends ActionRequest { private String id; private boolean verbose; private BytesReference source; + private XContentType xContentType = XContentType.JSON; + @Deprecated public SimulatePipelineRequest(BytesReference source) { if (source == null) { throw new IllegalArgumentException("source is missing"); } this.source = source; + this.xContentType = XContentFactory.xContentType(source); + } + + public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { + if (source == null) { + throw new IllegalArgumentException("source is missing"); + } + if (xContentType == null) { + throw new IllegalArgumentException("content type is missing"); + } + this.source = source; + this.xContentType = xContentType; } SimulatePipelineRequest() { @@ -78,12 +95,21 @@ public BytesReference getSource() { return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -92,6 +118,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } public static final class Fields { diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java index 4a13fa111e6a2..287a1c7217a94 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class SimulatePipelineRequestBuilder extends ActionRequestBuilder { @@ -29,10 +30,16 @@ public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipeli super(client, action, new SimulatePipelineRequest()); } + @Deprecated public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) { super(client, action, new SimulatePipelineRequest(source)); } + public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source, + XContentType xContentType) { + super(client, action, new SimulatePipelineRequest(source, xContentType)); + } + public SimulatePipelineRequestBuilder setId(String id) { request.setId(id); return this; diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java index 61fd400a1d372..3f67007df690d 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java @@ -47,7 +47,7 @@ public SimulatePipelineTransportAction(Settings settings, ThreadPool threadPool, @Override protected void doExecute(SimulatePipelineRequest request, ActionListener listener) { - final Map source = XContentHelper.convertToMap(request.getSource(), false).v2(); + final Map source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); final SimulatePipelineRequest.Parsed simulateRequest; try { diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 45e4aa8afdd42..4e2a7b466bc7c 100644 --- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -186,7 +186,8 @@ public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult()); if ((request.fetchSource() != null && request.fetchSource().fetchSource()) || (request.fields() != null && request.fields().length > 0)) { - Tuple> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true); + Tuple> sourceAndContent = + XContentHelper.convertToMap(upsertSourceBytes, true, upsertRequest.getContentType()); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes)); } else { update.setGetResult(null); diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 0d2801e44a003..84b0c4157c74e 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -29,8 +29,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -52,8 +50,6 @@ public class UpdateRequest extends InstanceShardOperationRequest implements DocWriteRequest, WriteRequest { - private static final DeprecationLogger DEPRECATION_LOGGER = - new DeprecationLogger(Loggers.getLogger(UpdateRequest.class)); private String type; private String id; @@ -559,7 +555,9 @@ public UpdateRequest doc(Map source, XContentType contentType) { /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(String, XContentType)} */ + @Deprecated public UpdateRequest doc(String source) { safeDoc().source(source); return this; @@ -568,6 +566,16 @@ public UpdateRequest doc(String source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(String source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source) { safeDoc().source(source); return this; @@ -576,11 +584,29 @@ public UpdateRequest doc(byte[] source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(byte[] source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source, int offset, int length) { safeDoc().source(source, offset, length); return this; } + /** + * Sets the doc to use for updates when a script is not specified. + */ + public UpdateRequest doc(byte[] source, int offset, int length, XContentType xContentType) { + safeDoc().source(source, offset, length, xContentType); + return this; + } + /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. @@ -644,7 +670,9 @@ public UpdateRequest upsert(Map source, XContentType contentType) { /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(String, XContentType)} */ + @Deprecated public UpdateRequest upsert(String source) { safeUpsertRequest().source(source); return this; @@ -653,6 +681,16 @@ public UpdateRequest upsert(String source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(String source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source) { safeUpsertRequest().source(source); return this; @@ -661,11 +699,29 @@ public UpdateRequest upsert(byte[] source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(byte[] source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source, int offset, int length) { safeUpsertRequest().source(source, offset, length); return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequest upsert(byte[] source, int offset, int length, XContentType xContentType) { + safeUpsertRequest().source(source, offset, length, xContentType); + return this; + } + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 50d84a241291f..ae33ad6e40986 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -223,7 +223,9 @@ public UpdateRequestBuilder setDoc(Map source, XContentType contentType) { /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setDoc(String source) { request.doc(source); return this; @@ -232,6 +234,16 @@ public UpdateRequestBuilder setDoc(String source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(String source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source) { request.doc(source); return this; @@ -240,11 +252,29 @@ public UpdateRequestBuilder setDoc(byte[] source) { /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) { request.doc(source, offset, length); return this; } + /** + * Sets the doc to use for updates when a script is not specified. + */ + public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) { + request.doc(source, offset, length, xContentType); + return this; + } + /** * Sets the doc to use for updates when a script is not specified. */ @@ -297,7 +327,9 @@ public UpdateRequestBuilder setUpsert(Map source, XContentType contentType) { /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setUpsert(String source) { request.upsert(source); return this; @@ -306,6 +338,16 @@ public UpdateRequestBuilder setUpsert(String source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source) { request.upsert(source); return this; @@ -314,11 +356,29 @@ public UpdateRequestBuilder setUpsert(byte[] source) { /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) { request.upsert(source, offset, length); return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) { + request.upsert(source, offset, length, xContentType); + return this; + } + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 14abc77513a13..3f705a215ea11 100644 --- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -112,6 +112,7 @@ import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; /** @@ -545,9 +546,16 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Stores an ingest pipeline + * @deprecated use {@link #preparePutPipeline(String, BytesReference, XContentType)} */ + @Deprecated PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source); + /** + * Stores an ingest pipeline + */ + PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType); + /** * Deletes a stored ingest pipeline */ @@ -596,8 +604,14 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Simulates an ingest pipeline */ + @Deprecated SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source); + /** + * Simulates an ingest pipeline + */ + SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType); + /** * Explain the allocation of a shard */ diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 075fbf1fad689..941da677a7be1 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -343,6 +343,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -1081,6 +1082,11 @@ public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference so return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source); } + @Override + public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType) { + return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source, xContentType); + } + @Override public void deletePipeline(DeletePipelineRequest request, ActionListener listener) { execute(DeletePipelineAction.INSTANCE, request, listener); @@ -1131,6 +1137,11 @@ public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference sou return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source); } + @Override + public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) { + return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source, xContentType); + } + @Override public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener listener) { execute(ClusterAllocationExplainAction.INSTANCE, request, listener); diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index e588b28b6354b..a3011284d25dd 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.InputStream; @@ -962,6 +963,21 @@ public Builder loadFromSource(String source) { return this; } + /** + * Loads settings from the actual string content that represents them using the + * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader + */ + public Builder loadFromSource(String source, XContentType xContentType) { + SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType); + try { + Map loadedSettings = settingsLoader.load(source); + put(loadedSettings); + } catch (Exception e) { + throw new SettingsException("Failed to load settings from [" + source + "]", e); + } + return this; + } + /** * Loads settings from a url that represents them using the * {@link SettingsLoaderFactory#loaderFromSource(String)}. diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java index 5f2da22c5f2dd..d454ca380e87a 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java +++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.settings.loader; +import org.elasticsearch.common.xcontent.XContentType; + /** * A class holding factory methods for settings loaders that attempts * to infer the type of the underlying settings content. @@ -33,9 +35,7 @@ private SettingsLoaderFactory() { * name. This factory method assumes that if the resource name ends * with ".json" then the content should be parsed as JSON, else if * the resource name ends with ".yml" or ".yaml" then the content - * should be parsed as YAML, else if the resource name ends with - * ".properties" then the content should be parsed as properties, - * otherwise default to attempting to parse as JSON. Note that the + * should be parsed as YAML, otherwise throws an exception. Note that the * parsers returned by this method will not accept null-valued * keys. * @@ -59,7 +59,7 @@ public static SettingsLoader loaderFromResource(String resourceName) { * contains an opening and closing brace ('{' and '}') then the * content should be parsed as JSON, else if the underlying content * fails this condition but contains a ':' then the content should - * be parsed as YAML, and otherwise should be parsed as properties. + * be parsed as YAML, and otherwise throws an exception. * Note that the JSON and YAML parsers returned by this method will * accept null-valued keys. * @@ -76,4 +76,19 @@ public static SettingsLoader loaderFromSource(String source) { } } + /** + * Returns a {@link SettingsLoader} based on the {@link XContentType} + * + * @param xContentType The content type + * @return A settings loader. + */ + public static SettingsLoader loaderFromXContentType(XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return new JsonSettingsLoader(true); + } else if (xContentType == XContentType.YAML) { + return new YamlSettingsLoader(true); + } else { + throw new IllegalArgumentException("unsupported content type [" + xContentType + "]"); + } + } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index df34ec726fd86..f9e14acdac1a4 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -969,16 +969,31 @@ public XContentBuilder rawField(String name, InputStream value) throws IOExcepti return this; } + public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + public XContentBuilder rawField(String name, BytesReference value) throws IOException { generator.writeRawField(name, value); return this; } + public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + public XContentBuilder rawValue(BytesReference value) throws IOException { generator.writeRawValue(value); return this; } + public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException { + generator.writeRawValue(value, contentType); + return this; + } + public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException { generator.copyCurrentStructure(parser); return this; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 478f3a8a08f82..94721cbc40a70 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -88,10 +88,16 @@ public interface XContentGenerator extends Closeable, Flushable { void writeRawField(String name, InputStream value) throws IOException; + void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException; + void writeRawField(String name, BytesReference value) throws IOException; + void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException; + void writeRawValue(BytesReference value) throws IOException; + void writeRawValue(BytesReference value, XContentType xContentType) throws IOException; + void copyCurrentStructure(XContentParser parser) throws IOException; /** diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index df96b2a974bd1..ea88828666ac5 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -55,10 +55,16 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry } } + @Deprecated public static Tuple> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { + return convertToMap(bytes, ordered, null); + } + + public static Tuple> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) + throws ElasticsearchParseException { try { - XContentType contentType; + XContentType contentType = xContentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -66,10 +72,14 @@ public static Tuple> convertToMap(BytesReferen if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); } - contentType = XContentFactory.xContentType(compressedStreamInput); + if (contentType == null) { + contentType = XContentFactory.xContentType(compressedStreamInput); + } input = compressedStreamInput; } else { - contentType = XContentFactory.xContentType(bytes); + if (contentType == null) { + contentType = XContentFactory.xContentType(bytes); + } input = bytes.streamInput(); } return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); @@ -105,15 +115,28 @@ public static Map convertToMap(XContent xContent, InputStream in } } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException { return convertToJson(bytes, reformatJson, false); } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - XContentType xContentType = XContentFactory.xContentType(bytes); + return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes)); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException { + return convertToJson(bytes, reformatJson, false, xContentType); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType) + throws IOException { if (xContentType == XContentType.JSON && !reformatJson) { return bytes.utf8ToString(); + } else if (xContentType == null) { + throw new IllegalArgumentException("the xcontent type must be provided"); } + // It is safe to use EMPTY here because this never uses namedObject try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY, bytes.streamInput())) { diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index ddd736e0d0070..e79713ee52097 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -40,7 +40,7 @@ public enum XContentType implements Writeable { */ JSON(0) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/json"; } @@ -64,7 +64,7 @@ public XContent xContent() { */ SMILE(1) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/smile"; } @@ -83,7 +83,7 @@ public XContent xContent() { */ YAML(2) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/yaml"; } @@ -102,7 +102,7 @@ public XContent xContent() { */ CBOR(3) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/cbor"; } @@ -133,6 +133,16 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) { return null; } + public static XContentType fromMediaTypeStrict(String mediaType) { + final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT); + for (XContentType type : values()) { + if (type.mediaTypeWithoutParameters().equals(mediaType)) { + return type; + } + } + return null; + } + private static boolean isSameMediaTypeAs(String stringType, XContentType type) { return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) || stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") || @@ -157,7 +167,7 @@ public String mediaType() { public abstract XContent xContent(); - protected abstract String mediaTypeWithoutParameters(); + public abstract String mediaTypeWithoutParameters(); public static XContentType readFrom(StreamInput in) throws IOException { int index = in.readVInt(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 07ae16b96c69b..1e09f8334f772 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -307,6 +307,11 @@ public void writeRawField(String name, InputStream content) throws IOException { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public void writeRawField(String name, InputStream content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { // EMPTY is safe here because we never call namedObject when writing raw data try (XContentParser parser = XContentFactory.xContent(contentType).createParser(NamedXContentRegistry.EMPTY, content)) { @@ -328,6 +333,11 @@ public final void writeRawField(String name, BytesReference content) throws IOEx if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public final void writeRawField(String name, BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { writeFieldName(name); copyRawValue(content, contentType.xContent()); @@ -345,6 +355,11 @@ public final void writeRawValue(BytesReference content) throws IOException { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawValue(content, contentType); + } + + @Override + public final void writeRawValue(BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { copyRawValue(content, contentType.xContent()); } else { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index fdc45530b9976..7e0f633e45dd2 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -256,8 +257,9 @@ public Map objectMappers() { return this.objectMappers; } + // TODO this method looks like it is only used in tests... public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException { - return parse(SourceToParse.source(index, type, id, source)); + return parse(SourceToParse.source(index, type, id, source, XContentFactory.xContentType(source))); } public ParsedDocument parse(SourceToParse source) throws MapperParsingException { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 0cafc50bbe2ab..55682c4acd330 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -23,15 +23,26 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; public class SourceToParse { public static SourceToParse source(String index, String type, String id, BytesReference source) { - return source(Origin.PRIMARY, index, type, id, source); + return source(Origin.PRIMARY, index, type, id, source, XContentFactory.xContentType(source)); + } + + public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) { + return source(Origin.PRIMARY, index, type, id, source, contentType); } public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) { - return new SourceToParse(origin, index, type, id, source); + return source(origin, index, type, id, source, XContentFactory.xContentType(source)); + } + + public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source, + XContentType contentType) { + return new SourceToParse(origin, index, type, id, source, contentType); } private final Origin origin; @@ -48,7 +59,9 @@ public static SourceToParse source(Origin origin, String index, String type, Str private String parentId; - private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) { + private XContentType xContentType; + + private SourceToParse(Origin origin, String index, String type, String id, BytesReference source, XContentType xContentType) { this.origin = Objects.requireNonNull(origin); this.index = Objects.requireNonNull(index); this.type = Objects.requireNonNull(type); @@ -56,6 +69,7 @@ private SourceToParse(Origin origin, String index, String type, String id, Bytes // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays this.source = new BytesArray(source.toBytesRef()); + this.xContentType = xContentType; } public Origin origin() { diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 5cb9f6111f5fe..90e7093d96ba5 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; @@ -29,7 +30,9 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Map; @@ -46,7 +49,9 @@ public final class PipelineConfiguration extends AbstractDiffable getParser() { @@ -56,6 +61,7 @@ private static class Builder { private String id; private BytesReference config; + private XContentType xContentType; void setId(String id) { this.id = id; @@ -65,8 +71,15 @@ void setConfig(BytesReference config) { this.config = config; } + void setXContentType(XContentType xContentType) { + this.xContentType = xContentType; + } + PipelineConfiguration build() { - return new PipelineConfiguration(id, config); + if (xContentType == null) { + xContentType = XContentFactory.xContentType(config); + } + return new PipelineConfiguration(id, config, xContentType); } } @@ -75,10 +88,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; + private final XContentType xContentType; - public PipelineConfiguration(String id, BytesReference config) { + public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { this.id = id; this.config = config; + this.xContentType = xContentType; } public String getId() { @@ -86,7 +101,17 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true).v2(); + return XContentHelper.convertToMap(config, true, xContentType).v2(); + } + + // pkg-private for tests + XContentType getXContentType() { + return xContentType; + } + + // pkg-private for tests + BytesReference getConfig() { + return config; } @Override @@ -99,7 +124,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference()); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in)); + } else { + final String id = in.readString(); + final BytesReference config = in.readBytesReference(); + return new PipelineConfiguration(id, config, XContentFactory.xContentType(config)); + } } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -110,6 +141,9 @@ public static Diff readDiffFrom(StreamInput in) throws IO public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java index 1171865a007d0..d476d7c2bd857 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java @@ -162,7 +162,7 @@ void validatePipeline(Map ingestInfos, PutPipelineReq throw new IllegalStateException("Ingest info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { @@ -185,7 +185,7 @@ ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metaData(MetaData.builder(currentState.getMetaData()) .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)) diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 4b3505e97e35c..73133fe86cb0f 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -58,28 +57,26 @@ protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.hasContent() ? request.content() : null, true); + return newBuilder(request.getXContentType(), true); } @Override public XContentBuilder newErrorBuilder() throws IOException { // Disable filtering when building error responses - return newBuilder(request.hasContent() ? request.content() : null, false); + return newBuilder(request.getXContentType(), false); } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { + public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { XContentType contentType = XContentType.fromMediaTypeOrFormat(format); if (contentType == null) { - // try and guess it from the auto detect source - if (autoDetectSource != null) { - contentType = XContentFactory.xContentType(autoDetectSource); + if (xContentType != null) { + contentType = xContentType; + } else { + // default to JSON output + contentType = XContentType.JSON; } } - if (contentType == null) { - // default to JSON - contentType = XContentType.JSON; - } Set includes = Collections.emptySet(); Set excludes = Collections.emptySet(); diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index 2a56313fd8a79..b8408071ef52c 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -35,7 +36,7 @@ public interface RestChannel { XContentBuilder newErrorBuilder() throws IOException; - XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException; + XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException; BytesStreamOutput bytesOutput(); diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 5ac82b7e4542e..73d590c140bf9 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -22,18 +22,19 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -42,8 +43,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.common.xcontent.XContentType; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.FORBIDDEN; @@ -165,15 +166,17 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont } RestChannel responseChannel = channel; try { - int contentLength = request.content().length(); - if (canTripCircuitBreaker(request)) { - inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); - } else { - inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + final int contentLength = request.hasContent() ? request.content().length() : 0; + if (checkContentType(request, responseChannel, contentLength)) { + if (canTripCircuitBreaker(request)) { + inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); + } else { + inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + } + // iff we could reserve bytes for the request we need to send the response also over this channel + responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); + dispatchRequest(request, responseChannel, client, threadContext); } - // iff we could reserve bytes for the request we need to send the response also over this channel - responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); - dispatchRequest(request, responseChannel, client, threadContext); } catch (Exception e) { try { responseChannel.sendResponse(new BytesRestResponse(channel, e)); @@ -214,6 +217,32 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final } } + boolean checkContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength) { + if (contentLength > 0) { + if (restRequest.getXContentType() == null && restRequest.isPlainText() == false) { + try { + XContentBuilder builder = channel.newErrorBuilder(); + final List contentTypeHeader = restRequest.getHeader("Content-Type"); + final String errorMessage; + if (contentTypeHeader == null) { + errorMessage = "Content-Type header is missing"; + } else { + errorMessage = "Content-Type header [" + + Strings.collectionToCommaDelimitedString(restRequest.getHeader("Content-Type")) + "] is not supported"; + } + builder.startObject().field("error", errorMessage).endObject(); + RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); + response.addHeader("Content-Type", builder.contentType().mediaType()); + channel.sendResponse(response); + } catch (IOException e) { + logger.warn("Failed to send response", e); + } + return false; + } + } + return true; + } + /** * Checks the request parameters against enabled settings for error trace support * @return true if the request does not have any parameters that conflict with system settings @@ -224,9 +253,9 @@ boolean checkRequestParameters(final RestRequest request, final RestChannel chan if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) { try { XContentBuilder builder = channel.newErrorBuilder(); - builder.startObject().field("error", "error traces in responses are disabled.").endObject().string(); + builder.startObject().field("error", "error traces in responses are disabled.").endObject(); RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); - response.addHeader("Content-Type", "application/json"); + response.addHeader("Content-Type", builder.contentType().mediaType()); channel.sendResponse(response); } catch (IOException e) { logger.warn("Failed to send response", e); @@ -312,8 +341,8 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - return delegate.newBuilder(autoDetectSource, useFiltering); + public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(xContentType, useFiltering); } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index 393e425baf937..2d33d6e3dda5d 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -28,7 +28,6 @@ public interface RestHandler { /** * Handles a rest request. - * * @param request The request to handle * @param channel The channel to write the request response to * @param client A client to use to make internal requests on behalf of the original request diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index b4b0614f8ad08..b4508f92b4efc 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -26,20 +26,24 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue; @@ -47,12 +51,25 @@ public abstract class RestRequest implements ToXContent.Params { + // tchar pattern as defined by RFC7230 section 3.2.6 + private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); + private final NamedXContentRegistry xContentRegistry; private final Map params; + private final Map> headers; private final String rawPath; private final Set consumedParams = new HashSet<>(); + private final XContentType xContentType; + private final boolean isPlainText; - public RestRequest(NamedXContentRegistry xContentRegistry, String uri) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param uri the URI of the request that potentially contains request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map> headers) { this.xContentRegistry = xContentRegistry; final Map params = new HashMap<>(); int pathEndPos = uri.indexOf('?'); @@ -63,12 +80,28 @@ public RestRequest(NamedXContentRegistry xContentRegistry, String uri) { RestUtils.decodeQueryString(uri, pathEndPos + 1, params); } this.params = params; + this.headers = Collections.unmodifiableMap(headers); + final List contentType = getHeader("Content-Type"); + this.xContentType = parseContentType(contentType); + this.isPlainText = xContentType == null && isPlainTextContentType(contentType); } - public RestRequest(NamedXContentRegistry xContentRegistry, Map params, String path) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param params the parameters of the request + * @param path the path of the request. This should not contain request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, Map params, String path, Map> headers) { this.xContentRegistry = xContentRegistry; this.params = params; this.rawPath = path; + this.headers = Collections.unmodifiableMap(headers); + final List contentType = getHeader("Content-Type"); + this.xContentType = parseContentType(contentType); + this.isPlainText = xContentType == null && isPlainTextContentType(contentType); } public enum Method { @@ -100,9 +133,52 @@ public final String path() { public abstract BytesReference content(); - public abstract String header(String name); + /** + * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are + * sent. Use of {@link #getHeader(String)} should be preferred + */ + public final String header(String name) { + List values = headers.get(name); + if (values != null && values.isEmpty() == false) { + return values.get(0); + } + return null; + } - public abstract Iterable> headers(); + /** + * Get all values for the header or {@code null} if the header was not found + */ + public final List getHeader(String name) { + List values = headers.get(name); + if (values != null) { + return Collections.unmodifiableList(values); + } + return null; + } + + /** + * Get all of the headers and values associated with the headers. Modifications of this map are not supported. + */ + public final Map> getHeaders() { + return headers; + } + + /** + * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of + * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request + * ({@link #isPlainText()}) + */ + @Nullable + public final XContentType getXContentType() { + return xContentType; + } + + /** + * Returns {@code true} iff the request has content and the {@code Content-Type} header is a plain text header + */ + public final boolean isPlainText() { + return isPlainText; + } @Nullable public SocketAddress getRemoteAddress() { @@ -254,8 +330,10 @@ public final XContentParser contentParser() throws IOException { BytesReference content = content(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); + } else if (xContentType == null) { + throw new IllegalStateException("Content-Type must be provided"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return xContentType.xContent().createParser(xContentRegistry, content); } /** @@ -283,11 +361,12 @@ public final boolean hasContentOrSourceParam() { * if you need to handle the absence request content gracefully. */ public final XContentParser contentOrSourceParamParser() throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return tuple.v1().xContent().createParser(xContentRegistry, content); } /** @@ -296,9 +375,11 @@ public final XContentParser contentOrSourceParamParser() throws IOException { * back to the user when there isn't request content. */ public final void withContentOrSourceParamParserOrNull(CheckedConsumer withParser) throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); if (content.length() > 0) { - try (XContentParser parser = XContentFactory.xContent(content).createParser(xContentRegistry, content)) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { withParser.accept(parser); } } else { @@ -310,7 +391,63 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { + if (hasContent()) { + if (xContentType == null) { + throw new IllegalStateException("Content-Type must be provided"); + } + return new Tuple<>(xContentType, content()); + } + String source = param("source"); + if (source != null) { + BytesArray bytes = new BytesArray(source); + String typeParam = param("source_type"); + final XContentType xContentType; + if (typeParam != null) { + xContentType = parseContentType(Collections.singletonList(typeParam)); + } else { + xContentType = XContentFactory.xContentType(bytes); + } + + if (xContentType == null) { + throw new IllegalStateException("could not determine source content type"); + } + return new Tuple<>(xContentType, bytes); + } + return new Tuple<>(XContentType.JSON, BytesArray.EMPTY); + } + + /** + * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source} + * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception + * back to the user when there isn't request content. This version allows for plain text content + */ + @Deprecated + public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer withParser) + throws IOException { + if (hasContent() && isPlainText) { + withParser.accept(null); + } + + Tuple tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); + if (content.length() > 0) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { + withParser.accept(parser); + } + } else { + withParser.accept(null); + } + } + + /** + * Get the content of the request or the contents of the {@code source} param without the xcontent type. This is useful the request can + * accept non xcontent values. + * @deprecated we should only take xcontent + */ + @Deprecated + public final BytesReference getContentOrSourceParamOnly() { if (hasContent()) { return content(); } @@ -320,4 +457,53 @@ public final BytesReference contentOrSourceParam() { } return BytesArray.EMPTY; } + + /** + * Parses the given content type string for the media type. This method currently ignores parameters. + */ + // TODO stop ignoring parameters such as charset... + private static XContentType parseContentType(List header) { + if (header == null || header.isEmpty()) { + return null; + } else if (header.size() > 1) { + throw new IllegalArgumentException("only one Content-Type header should be provided"); + } + + String rawContentType = header.get(0); + final String[] elements = rawContentType.split("[ \t]*;"); + if (elements.length > 0) { + final String[] splitMediaType = elements[0].split("/"); + if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() + && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { + return XContentType.fromMediaTypeStrict(elements[0]); + } else { + throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); + } + } + return null; + } + + /** + * Parses the Content-Type header and returns true if the header matches the plain text media type + */ + private static boolean isPlainTextContentType(List header) { + if (header == null || header.isEmpty()) { + return false; + } else if (header.size() > 1) { + throw new IllegalArgumentException("only one Content-Type header should be provided"); + } + + String rawContentType = header.get(0); + final String[] elements = rawContentType.split("[ \t]*;"); + if (elements.length > 0) { + final String[] splitMediaType = elements[0].split("/"); + if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() + && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { + return splitMediaType[0].equals("text") && (splitMediaType[1].equals("plain") || splitMediaType[1].equals("plain;")); + } else { + throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); + } + } + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index aec998e6f9a22..05bbb5202e62d 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -54,7 +54,7 @@ protected String getScriptLang(RestRequest request) { @Override public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException { PutStoredScriptRequest putRequest = new PutStoredScriptRequest(getScriptLang(request), request.param("id")); - putRequest.script(request.content()); + putRequest.script(request.content(), request.getXContentType()); return channel -> client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index d1e6919f4c06a..f83e107b6ea6e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -45,7 +45,7 @@ public RestCreateIndexAction(Settings settings, RestController controller) { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { - createIndexRequest.source(request.content()); + createIndexRequest.source(request.content(), request.getXContentType()); } createIndexRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index fd8a56b4df1e7..5d602c2560214 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -59,7 +59,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout())); putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - putRequest.source(request.content()); + putRequest.source(request.content(), request.getXContentType()); return channel -> client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<>(channel)); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 0fa394e6f3d65..f85239824c868 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; @@ -69,11 +68,11 @@ public RestPutMappingAction(Settings settings, RestController controller) { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); putMappingRequest.type(request.param("type")); - putMappingRequest.source(request.content().utf8ToString()); + putMappingRequest.source(request.content().utf8ToString(), request.getXContentType()); putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); - return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener(channel)); + return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java index 4c6730d5b1596..6332028ff5d4b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java @@ -68,7 +68,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Settings.Builder updateSettings = Settings.builder(); String bodySettingsStr = request.content().utf8ToString(); if (Strings.hasText(bodySettingsStr)) { - Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build(); + Settings buildSettings = Settings.builder() + .loadFromSource(bodySettingsStr, request.getXContentType()) + .build(); for (Map.Entry entry : buildSettings.getAsMap().entrySet()) { String key = entry.getKey(); String value = entry.getValue(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 46731a5296090..069e7cd06341e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -95,7 +95,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, - defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex); + defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestBuilderListener(channel) { @Override diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index ed36cc3c89d87..26a2f0862fd27 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -69,7 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC client.get(getRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(GetResponse response) throws Exception { - XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false); + XContentBuilder builder = channel.newBuilder(request.getXContentType(), false); if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing return new BytesRestResponse(NOT_FOUND, builder); } else { diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 43739976ba084..23304662ec02b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -68,7 +68,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC indexRequest.routing(request.param("routing")); indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing indexRequest.setPipeline(request.param("pipeline")); - indexRequest.source(request.content()); + indexRequest.source(request.content(), request.getXContentType()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java index 08bd65f68922f..37241e4fb1e5e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java @@ -21,8 +21,11 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -42,7 +45,8 @@ public RestPutPipelineAction(Settings settings, RestController controller) { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), restRequest.contentOrSourceParam()); + Tuple sourceTuple = restRequest.contentOrSourceParam(); + PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout())); request.timeout(restRequest.paramAsTime("timeout", request.timeout())); return channel -> client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java index eb29be1994fc4..64aa257db761e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -21,8 +21,11 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -44,7 +47,8 @@ public RestSimulatePipelineAction(Settings settings, RestController controller) @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - SimulatePipelineRequest request = new SimulatePipelineRequest(restRequest.contentOrSourceParam()); + Tuple sourceTuple = restRequest.contentOrSourceParam(); + SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1()); request.setId(restRequest.param("id")); request.setVerbose(restRequest.paramAsBoolean("verbose", false)); return channel -> client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java index 675ba25d5d740..c2a983e35e0b8 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java @@ -20,13 +20,11 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -53,26 +51,29 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC String scrollIds = request.param("scroll_id"); ClearScrollRequest clearRequest = new ClearScrollRequest(); clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - scrollIds = body.utf8ToString(); - clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); + request.withContentOrSourceParamParserOrNullLenient((xContentParser -> { + if (xContentParser == null) { + if (request.hasContent() && request.isPlainText()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.content(); + String bodyScrollIds = body.utf8ToString(); + clearRequest.setScrollIds(Arrays.asList(splitScrollIds(bodyScrollIds))); + } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value clearRequest.setScrollIds(null); - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, clearRequest); + try { + buildFromContent(xContentParser, clearRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + })); - return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener(channel)); + return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel)); } - public static String[] splitScrollIds(String scrollIds) { + private static String[] splitScrollIds(String scrollIds) { if (scrollIds == null) { return Strings.EMPTY_ARRAY; } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index a5a56ebe543d2..39a99f1fd89a7 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -26,11 +26,12 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -95,7 +96,7 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a } /** - * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer. + * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer. */ public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex, BiConsumer consumer) throws IOException { @@ -105,9 +106,10 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind String searchType = request.param("search_type"); String routing = request.param("routing"); - final BytesReference data = request.contentOrSourceParam(); + final Tuple sourceTuple = request.contentOrSourceParam(); + final XContent xContent = sourceTuple.v1().xContent(); + final BytesReference data = sourceTuple.v2(); - XContent xContent = XContentFactory.xContent(data); int from = 0; int length = data.length(); byte marker = xContent.streamSeparator(); @@ -178,7 +180,7 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind break; } BytesReference bytes = data.slice(from, nextMarker - from); - try (XContentParser parser = XContentFactory.xContent(bytes).createParser(request.getXContentRegistry(), bytes)) { + try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) { consumer.accept(searchRequest, parser); } // move pointers diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index 5a8e5f8efe548..c090c7d8bc784 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -61,22 +60,25 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll"))); } - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - if (scrollId == null) { - scrollId = body.utf8ToString(); - searchScrollRequest.scrollId(scrollId); + request.withContentOrSourceParamParserOrNull(xContentParser -> { + if (xContentParser == null) { + if (request.hasContent() && request.isPlainText()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.getContentOrSourceParamOnly(); + if (scrollId == null) { + String bodyScrollId = body.utf8ToString(); + searchScrollRequest.scrollId(bodyScrollId); + } } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, searchScrollRequest); + try { + buildFromContent(xContentParser, searchScrollRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + }); return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java index 7c99f3a80ddf9..f4b7ef86341f9 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -70,14 +70,15 @@ public String getScript(String language, String id) { return scriptAsBytes.utf8ToString(); } - public static String parseStoredScript(BytesReference scriptAsBytes) { + public static String parseStoredScript(BytesReference scriptAsBytes, XContentType xContentType) { // Scripts can be stored via API in several ways: // 1) wrapped into a 'script' json object or field // 2) wrapped into a 'template' json object or field // 3) just as is // In order to fetch the actual script in consistent manner this parsing logic is needed: // EMPTY is ok here because we never call namedObject, we're just copying structure. - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, scriptAsBytes); + try (XContentParser parser = xContentType == null ? XContentHelper.createParser(NamedXContentRegistry.EMPTY, scriptAsBytes) : + xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, scriptAsBytes); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { parser.nextToken(); parser.nextToken(); @@ -215,8 +216,8 @@ public Builder(ScriptMetaData previous) { } } - public Builder storeScript(String lang, String id, BytesReference script) { - BytesReference scriptBytest = new BytesArray(parseStoredScript(script)); + public Builder storeScript(String lang, String id, BytesReference script, XContentType xContentType) { + BytesReference scriptBytest = new BytesArray(parseStoredScript(script, xContentType)); scripts.put(toKey(lang, id), new ScriptAsBytes(scriptBytest)); return this; } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 60cdb03e47909..ab1d11408ca03 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -55,6 +55,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.env.Environment; import org.elasticsearch.search.lookup.SearchLookup; @@ -376,9 +377,9 @@ String getScriptFromClusterState(String scriptLang, String id) { return script; } - void validateStoredScript(String id, String scriptLang, BytesReference scriptBytes) { + void validateStoredScript(String id, String scriptLang, BytesReference scriptBytes, XContentType xContentType) { validateScriptSize(id, scriptBytes.length()); - String script = ScriptMetaData.parseStoredScript(scriptBytes); + String script = ScriptMetaData.parseStoredScript(scriptBytes, xContentType); if (Strings.hasLength(scriptBytes)) { //Just try and compile it try { @@ -411,7 +412,7 @@ void validateStoredScript(String id, String scriptLang, BytesReference scriptByt public void storeScript(ClusterService clusterService, PutStoredScriptRequest request, ActionListener listener) { String scriptLang = validateScriptLanguage(request.scriptLang()); //verify that the script compiles - validateStoredScript(request.id(), scriptLang, request.script()); + validateStoredScript(request.id(), scriptLang, request.script(), request.xContentType()); clusterService.submitStateUpdateTask("put-script-" + request.id(), new AckedClusterStateUpdateTask(request, listener) { @Override @@ -429,7 +430,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { static ClusterState innerStoreScript(ClusterState currentState, String validatedScriptLang, PutStoredScriptRequest request) { ScriptMetaData scriptMetadata = currentState.metaData().custom(ScriptMetaData.TYPE); ScriptMetaData.Builder scriptMetadataBuilder = new ScriptMetaData.Builder(scriptMetadata); - scriptMetadataBuilder.storeScript(validatedScriptLang, request.id(), request.script()); + scriptMetadataBuilder.storeScript(validatedScriptLang, request.id(), request.script(), request.xContentType()); MetaData.Builder metaDataBuilder = MetaData.builder(currentState.getMetaData()) .putCustom(ScriptMetaData.TYPE, scriptMetadataBuilder.build()); return ClusterState.builder(currentState).metaData(metaDataBuilder).build(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 7f490ebab9050..580d8149fd5bc 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -300,7 +301,8 @@ public void testTransportBulkTasks() { registerTaskManageListeners(BulkAction.NAME + "[s][r]"); // shard task on replica createIndex("test"); ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks - client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}")).get(); + client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id") + .setSource("{\"foo\": \"bar\"}", XContentType.JSON)).get(); // the bulk operation should produce one main task List topTask = findEvents(BulkAction.NAME, Tuple::v1); @@ -352,7 +354,7 @@ public void testSearchTaskDescriptions() { registerTaskManageListeners(SearchAction.NAME + "[*]"); // shard task createIndex("test"); ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks - client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}") + client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}", XContentType.JSON) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertSearchResponse(client().prepareSearch("test").setTypes("doc").setQuery(QueryBuilders.matchAllQuery()).get()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 546b1435c3513..4b531267d3ae9 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -209,7 +209,7 @@ private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOExcep FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry()); requestBuilder.withParams(params); if (hasBody) { - requestBuilder.withContent(builder.bytes()); + requestBuilder.withContent(builder.bytes(), builder.contentType()); } return requestBuilder.build(); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java new file mode 100644 index 0000000000000..7716888a92482 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class PutStoredScriptRequestTests extends ESTestCase { + + public void testSerialization() throws IOException { + PutStoredScriptRequest storedScriptRequest = new PutStoredScriptRequest("foo", "bar"); + storedScriptRequest.script(new BytesArray("{}"), XContentType.JSON); + + assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + BytesStreamOutput output = new BytesStreamOutput(); + storedScriptRequest.writeTo(output); + + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + PutStoredScriptRequest serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + assertEquals(storedScriptRequest.scriptLang(), serialized.scriptLang()); + assertEquals(storedScriptRequest.id(), serialized.id()); + + // send to an old version and then read it + output = new BytesStreamOutput(); + output.setVersion(Version.V_5_0_0); + storedScriptRequest.writeTo(output); + in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + assertEquals(storedScriptRequest.scriptLang(), serialized.scriptLang()); + assertEquals(storedScriptRequest.id(), serialized.id()); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java index ad17689bb789c..c3b2eb80584b5 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.junit.After; @@ -57,7 +58,7 @@ public void tearDown() throws Exception { */ public void testSetSource() throws IOException { CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE); - builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}"); + builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); XContentBuilder xContent = XContentFactory.jsonBuilder().startObject().field(KEY, VALUE).endObject(); @@ -68,7 +69,7 @@ public void testSetSource() throws IOException { ByteArrayOutputStream docOut = new ByteArrayOutputStream(); XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field(KEY, VALUE).endObject(); doc.close(); - builder.setSource(docOut.toByteArray()); + builder.setSource(docOut.toByteArray(), XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); Map settingsMap = new HashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index e19f930a22d22..9ab6551d6fd07 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -59,7 +60,8 @@ public void testCreateShrinkIndexToN() { internalCluster().ensureAtLeastNumDataNodes(2); prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); @@ -85,7 +87,8 @@ public void testCreateShrinkIndexToN() { assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); for (int i = 0; i < 20; i++) { // now update - client().prepareIndex("first_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("first_shrink", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); @@ -113,7 +116,8 @@ public void testCreateShrinkIndexToN() { assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); for (int i = 0; i < 20; i++) { // now update - client().prepareIndex("second_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("second_shrink", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20); @@ -129,7 +133,8 @@ public void testCreateShrinkIndex() { .put("index.version.created", version) ).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); @@ -164,7 +169,8 @@ public void testCreateShrinkIndex() { } for (int i = 20; i < 40; i++) { - client().prepareIndex("target", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("target", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } flushAndRefresh(); assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40); @@ -181,7 +187,8 @@ public void testCreateShrinkIndexFails() throws Exception { .put("number_of_shards", randomIntBetween(2, 7)) .put("number_of_replicas", 0)).get(); for (int i = 0; i < 20; i++) { - client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get(); + client().prepareIndex("source", randomFrom("t1", "t2", "t3")) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); } ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 9c93e5c73d9f5..6fea38d0e2821 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -19,10 +19,17 @@ package org.elasticsearch.action.admin.indices.mapping.put; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + public class PutMappingRequestTests extends ESTestCase { public void testValidation() { @@ -41,12 +48,12 @@ public void testValidation() { assertNotNull("source validation should fail", ex); assertTrue(ex.getMessage().contains("source is missing")); - r.source(""); + r.source("", XContentType.JSON); ex = r.validate(); assertNotNull("source validation should fail", ex); assertTrue(ex.getMessage().contains("source is empty")); - r.source("somevalidmapping"); + r.source("somevalidmapping", XContentType.JSON); ex = r.validate(); assertNull("validation should succeed", ex); @@ -64,4 +71,42 @@ public void testBuildFromSimplifiedDef() { () -> PutMappingRequest.buildFromSimplifiedDef("type", "only_field")); assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage()); } + + public void testPutMappingRequestFromOldVersion() throws IOException { + // this is hacky but here goes + PutMappingRequest request = new PutMappingRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.source(mapping, XContentType.JSON); // THIS IS NOT A BUG! Intentionally specifying the wrong type so we serialize it + assertEquals(mapping, request.source()); + + // output version doesn't matter + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + request.writeTo(bytesStreamOutput); + + StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + PutMappingRequest serialized = new PutMappingRequest(); + serialized.readFrom(in); + + // yaml is translated to JSON on reading + String source = serialized.source(); + assertNotEquals(mapping, source); + assertTrue(source.startsWith("{")); + + // reading from a fixed version does no translation + in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); + assertEquals(Version.CURRENT, in.getVersion()); + serialized = new PutMappingRequest(); + serialized.readFrom(in); + assertEquals(mapping, serialized.source()); + } + + public void testPutMappingRequestTranslatesNonJsonToJson() throws IOException { + // this is hacky but here goes + PutMappingRequest request = new PutMappingRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.source(mapping, XContentType.YAML); + assertNotEquals(mapping, request.source()); + assertTrue(request.source().startsWith("{")); + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java index 7ea103313fe83..d098dd6befaa2 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.template; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESSingleNodeTestCase; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -33,10 +34,10 @@ public void testBeatsTemplatesBWC() throws Exception { String packetBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json"); String fileBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json"); String winLogBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json"); - client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat).get(); - client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat).get(); - client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat).get(); - client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat).get(); + client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get(); + client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat, XContentType.JSON).get(); client().prepareIndex("metricbeat-foo", "doc", "1").setSource("message", "foo").get(); client().prepareIndex("packetbeat-foo", "doc", "1").setSource("message", "foo").get(); @@ -47,7 +48,7 @@ public void testBeatsTemplatesBWC() throws Exception { public void testLogstashTemplatesBWC() throws Exception { String ls5x = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json"); - client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x).get(); + client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get(); client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get(); assertWarnings("Deprecated field [template] used, replaced by [index_patterns]"); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index 076245ad76d5b..00769f692e559 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -22,6 +22,8 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -66,4 +68,39 @@ public void testPutIndexTemplateRequest510() throws IOException { } } + public void testPutIndexTemplateRequestSerializationXContent() throws IOException { + PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.patterns(Collections.singletonList("foo")); + // THIS IS NOT A BUG! Intentionally specifying the wrong type so we serialize it + request.mapping("bar", mapping, XContentType.JSON); + assertEquals(mapping, request.mappings().get("bar")); + + BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(Version.V_5_0_0); + request.writeTo(out); + + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); + serialized.readFrom(in); + assertNotEquals(mapping, serialized.mappings().get("bar")); + assertTrue(serialized.mappings().get("bar").startsWith("{")); + + out = new BytesStreamOutput(); + request.writeTo(out); + in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + serialized = new PutIndexTemplateRequest(); + serialized.readFrom(in); + assertEquals(mapping, serialized.mappings().get("bar")); + } + + public void testContentTypesAreConvertedWhenSettingMapping() throws IOException { + PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + request.mapping("bar", mapping, XContentType.YAML); + assertNotEquals(mapping, request.mappings().get("bar")); + assertFalse(mapping.startsWith("{")); + assertTrue(request.mappings().get("bar").startsWith("{")); + } } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 4300a629fbd8c..22377ea1769c8 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import java.nio.charset.StandardCharsets; @@ -31,7 +32,7 @@ public class BulkIntegrationIT extends ESIntegTestCase { public void testBulkIndexCreatesMapping() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/bulk-log.json"); BulkRequestBuilder bulkBuilder = client().prepareBulk(); - bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); bulkBuilder.get(); assertBusy(new Runnable() { @Override diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index 48d989209cbba..ca1630b00641c 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -36,9 +37,9 @@ public void testBulkProcessorAutoCreateRestrictions() throws Exception { client().admin().cluster().prepareHealth("willwork").setWaitForGreenStatus().execute().actionGet(); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}")); - bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}")); - bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}")); + bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}", XContentType.JSON)); BulkResponse br = bulkRequestBuilder.get(); BulkItemResponse[] responses = br.getItems(); assertEquals(3, responses.length); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java index 4a41499d0d127..b6242e6d5fcd4 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -44,7 +45,7 @@ public void testBulkRequestModifier() { int numRequests = scaledRandomIntBetween(8, 64); BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { - bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}")); + bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}", XContentType.JSON)); } CaptureActionListener actionListener = new CaptureActionListener(); TransportBulkAction.BulkRequestModifier bulkRequestModifier = new TransportBulkAction.BulkRequestModifier(bulkRequest); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index a4a5f6f5ba658..4f53000d16821 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.Script; import org.elasticsearch.test.ESTestCase; @@ -55,7 +56,7 @@ public void testSimpleBulk1() throws Exception { bulkAction = Strings.replace(bulkAction, "\r\n", "\n"); } BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }"))); assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class)); @@ -65,21 +66,21 @@ public void testSimpleBulk1() throws Exception { public void testSimpleBulk2() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } public void testSimpleBulk3() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk3.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } public void testSimpleBulk4() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk4.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(4)); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1")); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2)); @@ -101,14 +102,14 @@ public void testSimpleBulk4() throws Exception { public void testBulkAllowExplicitIndex() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); try { - new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false); + new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false, XContentType.JSON); fail(); } catch (Exception e) { } bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk5.json"); - new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false); + new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false, XContentType.JSON); } public void testBulkAddIterable() { @@ -128,7 +129,7 @@ public void testSimpleBulk6() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk6.json"); BulkRequest bulkRequest = new BulkRequest(); ParsingException exc = expectThrows(ParsingException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Unknown key for a VALUE_STRING in [hello]")); } @@ -136,7 +137,7 @@ public void testSimpleBulk7() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk7.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Malformed action/metadata line [5], expected a simple value for field [_unkown] but found [START_ARRAY]")); } @@ -145,7 +146,7 @@ public void testSimpleBulk8() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk8.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Action/metadata line [3] contains an unknown parameter [_foo]")); } @@ -153,14 +154,14 @@ public void testSimpleBulk9() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk9.json"); BulkRequest bulkRequest = new BulkRequest(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null)); + () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON)); assertThat(exc.getMessage(), containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]")); } public void testSimpleBulk10() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk10.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); assertThat(bulkRequest.numberOfActions(), equalTo(9)); } @@ -172,8 +173,8 @@ public void testBulkRequestWithRefresh() throws Exception { // We force here a "type is missing" validation error bulkRequest.add(new DeleteRequest("index", null, "id")); bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); - bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new IndexRequest("index", "type", "id").source("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); ActionRequestValidationException validate = bulkRequest.validate(); assertThat(validate, notNullValue()); assertThat(validate.validationErrors(), not(empty())); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 82eee3554e804..e87107b7fc00d 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -493,8 +494,8 @@ public void testBulkUpdateDocAsUpsertWithParent() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChild, 0, addChild.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChild, 0, addChild.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(2)); @@ -576,9 +577,9 @@ public void testBulkUpdateUpsertWithParent() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChild1, 0, addChild1.length); - builder.add(addChild2, 0, addChild2.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChild1, 0, addChild1.length, XContentType.JSON); + builder.add(addChild2, 0, addChild2.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(3)); @@ -650,10 +651,10 @@ public void testBulkUpdateChildMissingParentRouting() throws Exception { "}" + "\n").array(); - builder.add(addParent, 0, addParent.length); - builder.add(addChildOK, 0, addChildOK.length); - builder.add(addChildMissingRouting, 0, addChildMissingRouting.length); - builder.add(addChildOK, 0, addChildOK.length); + builder.add(addParent, 0, addParent.length, XContentType.JSON); + builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON); + builder.add(addChildMissingRouting, 0, addChildMissingRouting.length, XContentType.JSON); + builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON); BulkResponse bulkResponse = builder.get(); assertThat(bulkResponse.getItems().length, equalTo(4)); @@ -733,19 +734,19 @@ public void testThatInvalidIndexNamesShouldNotBreakCompleteBulkRequest() { // issue 6630 public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { BulkResponse indexBulkItemResponse = client().prepareBulk() - .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }")) - .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }")) - .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }")) - .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }")) + .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }", XContentType.JSON)) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); assertNoFailures(indexBulkItemResponse); BulkResponse bulkItemResponse = client().prepareBulk() - .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }")) - .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }")) - .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}")) - .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}")) + .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }", XContentType.JSON)) + .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }", XContentType.JSON)) + .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}", XContentType.JSON)) + .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}", XContentType.JSON)) .add(new DeleteRequest("test", "type", "5")) .add(new DeleteRequest("test", "type", "6")) .get(); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 29c55c426d313..cdfe54575fa9b 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; @@ -174,7 +175,7 @@ private void runTestTook(boolean controlled) throws Exception { bulkAction = Strings.replace(bulkAction, "\r\n", "\n"); } BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON); AtomicLong expected = new AtomicLong(); TransportBulkAction action = createAction(controlled, expected); action.doExecute(null, bulkRequest, new ActionListener() { diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java index 06e9d586e3671..5341588f7226b 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java @@ -74,8 +74,9 @@ public void testSetSource() throws Exception { ByteArrayOutputStream docOut = new ByteArrayOutputStream(); XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field("SomeKey", "SomeValue").endObject(); doc.close(); - indexRequestBuilder.setSource(docOut.toByteArray()); - assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true)); + indexRequestBuilder.setSource(docOut.toByteArray(), XContentType.JSON); + assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true, + indexRequestBuilder.request().getContentType())); doc = XContentFactory.jsonBuilder().startObject().field("SomeKey", "SomeValue").endObject(); doc.close(); diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index a722529626843..bd2fc5bc335e8 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -18,16 +18,21 @@ */ package org.elasticsearch.action.index; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -83,19 +88,19 @@ public void testCreateOperationRejectsVersions() { public void testIndexingRejectsLongIds() { String id = randomAsciiOfLength(511); IndexRequest request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); ActionRequestValidationException validate = request.validate(); assertNull(validate); id = randomAsciiOfLength(512); request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); validate = request.validate(); assertNull(validate); id = randomAsciiOfLength(513); request = new IndexRequest("index", "type", id); - request.source("{}"); + request.source("{}", XContentType.JSON); validate = request.validate(); assertThat(validate, notNullValue()); assertThat(validate.getMessage(), @@ -150,4 +155,33 @@ public void testIndexResponse() { ",shards={\"total\":" + total + ",\"successful\":" + successful + ",\"failed\":0}]", indexResponse.toString()); } + + public void testIndexRequestXContentSerialization() throws IOException { + IndexRequest indexRequest = new IndexRequest("foo", "bar", "1"); + indexRequest.source("{}", XContentType.JSON); + assertEquals(XContentType.JSON, indexRequest.getContentType()); + + BytesStreamOutput out = new BytesStreamOutput(); + indexRequest.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getContentType()); + + // test with an incorrect content type and send it to an old version then see that we get the right content type when reading + indexRequest = new IndexRequest("foo", "bar", "1"); + indexRequest.source("{}", XContentType.YAML); + assertEquals(XContentType.YAML, indexRequest.getContentType()); + + out = new BytesStreamOutput(); + out.setVersion(Version.V_5_0_0); + indexRequest.writeTo(out); + in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + + serialized = new IndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getContentType()); + assertEquals("{}", serialized.source().utf8ToString()); + } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java new file mode 100644 index 0000000000000..33a1be5f337f4 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.ingest; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class PutPipelineRequestTests extends ESTestCase { + + public void testSerializationWithXContent() throws IOException { + PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, request.getXContentType()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + PutPipelineRequest serialized = new PutPipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + + // send to old and read from old with a bad content type and see that we find the correct one + request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); + assertEquals(XContentType.YAML, request.getXContentType()); + + output = new BytesStreamOutput(); + output.setVersion(Version.V_5_0_0); + request.writeTo(output); + in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + + serialized = new PutPipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java index 0966010a8f79a..bdc7efa92c049 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java @@ -19,19 +19,22 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.nio.charset.StandardCharsets; import static org.hamcrest.CoreMatchers.equalTo; public class SimulatePipelineRequestTests extends ESTestCase { public void testSerialization() throws IOException { - SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray("")); + SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""), XContentType.JSON); // Sometimes we set an id if (randomBoolean()) { request.setId(randomAsciiOfLengthBetween(1, 10)); @@ -51,4 +54,34 @@ public void testSerialization() throws IOException { assertThat(otherRequest.getId(), equalTo(request.getId())); assertThat(otherRequest.isVerbose(), equalTo(request.isVerbose())); } + + public void testSerializationWithXContent() throws IOException { + SimulatePipelineRequest request = + new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, request.getXContentType()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + SimulatePipelineRequest serialized = new SimulatePipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + + // send to old and read from old with a bad content type and see that we find the correct one + request = new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); + assertEquals(XContentType.YAML, request.getXContentType()); + + output = new BytesStreamOutput(); + output.setVersion(Version.V_5_0_0); + request.writeTo(output); + in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + + serialized = new SimulatePipelineRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getSource().utf8ToString()); + } } diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index d249c33edcc6f..d1de2f7ccf756 100644 --- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; @@ -160,7 +161,8 @@ public void testMaxConcurrentSearchRequests() { private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException { byte[] data = StreamsUtils.copyToBytesFromClasspath(sample); - RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(data)).build(); + RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray(data), XContentType.JSON).build(); return RestMultiSearchAction.parseRequest(restRequest, true); } diff --git a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java index 2e1a00afc2048..cab27d74c7d58 100644 --- a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java +++ b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -44,9 +45,9 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertAcked(createIndexResponse); // indexing, by default, will work (waiting for one shard copy only) - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON).execute().actionGet(); try { - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(2) // wait for 2 active shard copies .setTimeout(timeValueMillis(100)).execute().actionGet(); fail("can't index, does not enough active shard copies"); @@ -70,12 +71,12 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // this should work, since we now have two - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(2) .setTimeout(timeValueSeconds(1)).execute().actionGet(); try { - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) .setTimeout(timeValueMillis(100)).execute().actionGet(); fail("can't index, not enough active shard copies"); @@ -92,7 +93,7 @@ public void testReplicationWaitsForActiveShardCount() throws Exception { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); // this should work, since we now have all shards started - client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) + client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON) .setWaitForActiveShards(ActiveShardCount.ALL) .setTimeout(timeValueSeconds(1)).execute().actionGet(); } diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 892401356c34e..e1133542e4102 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -144,7 +144,8 @@ public void testUpdateRequest() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); - Map upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2(); + Map upsertDoc = + XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); @@ -171,7 +172,7 @@ public void testUpdateRequest() throws Exception { assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); assertThat(params.get("param1").toString(), equalTo("value1")); - upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2(); + upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2(); assertThat(upsertDoc.get("field1").toString(), equalTo("value1")); assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2")); diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 9f04f74cd2480..497b134ebe386 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; @@ -90,7 +91,8 @@ public void testAliases() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1")); logger.info("--> indexing against [alias1], should work now"); - IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet(); + IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getIndex(), equalTo("test")); logger.info("--> creating index [test_x]"); @@ -102,7 +104,8 @@ public void testAliases() throws Exception { assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1")); logger.info("--> indexing against [alias1], should work against [test_x]"); - indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet(); + indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getIndex(), equalTo("test_x")); } @@ -164,15 +167,14 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test"))); logger.info("--> indexing against [test]"); - client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); - client().index( - indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) - .actionGet(); + client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); + client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else"), XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet(); logger.info("--> checking single filtering alias search"); SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get(); @@ -254,16 +256,16 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test2", "foos", termQuery("name", "foo"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get(); - client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get(); - client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get(); - client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get(); + client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); logger.info("--> indexing against [test2]"); - client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get(); - client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get(); - client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get(); - client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get(); + client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get(); refresh(); @@ -322,17 +324,17 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test3", "filter13", termQuery("name", "baz"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"))).get(); - client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"))).get(); - client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"))).get(); + client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"), XContentType.JSON)).get(); - client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"))).get(); - client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"))).get(); - client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"))).get(); + client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"), XContentType.JSON)).get(); - client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"))).get(); - client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"))).get(); - client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"))).get(); + client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"), XContentType.JSON)).get(); + client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"), XContentType.JSON)).get(); + client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"), XContentType.JSON)).get(); refresh(); @@ -382,16 +384,16 @@ public void testDeletingByQueryFilteringAliases() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test2", "tests", termQuery("name", "test"))); logger.info("--> indexing against [test1]"); - client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get(); - client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get(); - client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get(); - client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get(); + client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get(); logger.info("--> indexing against [test2]"); - client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get(); - client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get(); - client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get(); - client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get(); + client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get(); + client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get(); refresh(); @@ -435,7 +437,7 @@ public void testWaitForAliasCreationMultipleShards() throws Exception { for (int i = 0; i < 10; i++) { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i)); - client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get(); + client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get(); } } @@ -447,7 +449,7 @@ public void testWaitForAliasCreationSingleShard() throws Exception { for (int i = 0; i < 10; i++) { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i)); - client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get(); + client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get(); } } @@ -466,7 +468,8 @@ public void testWaitForAliasSimultaneousUpdate() throws Exception { @Override public void run() { assertAcked(admin().indices().prepareAliases().addAlias("test", aliasName)); - client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"))).actionGet(); + client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"), XContentType.JSON)) + .actionGet(); } }); } @@ -745,7 +748,7 @@ public void testCreateIndexWithAliasesInSource() throws Exception { " \"alias2\" : {\"filter\" : {\"match_all\": {}}},\n" + " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" + " }\n" + - "}")); + "}", XContentType.JSON)); checkAliases(); } @@ -826,8 +829,8 @@ public void testAliasesFilterWithHasChildQuery() throws Exception { .addMapping("parent") .addMapping("child", "_parent", "type=parent") ); - client().prepareIndex("my-index", "parent", "1").setSource("{}").get(); - client().prepareIndex("my-index", "child", "2").setSource("{}").setParent("1").get(); + client().prepareIndex("my-index", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("my-index", "child", "2").setSource("{}", XContentType.JSON).setParent("1").get(); refresh(); assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", hasChildQuery("child", matchAllQuery(), ScoreMode.None))); diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java index ccc72db9d7a5f..d3a0e0b3e4e14 100644 --- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java +++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -106,7 +107,7 @@ public void testActions() { client.prepareSearch().execute().addListener(new AssertingActionListener<>(SearchAction.NAME, client.threadPool())); client.prepareDelete("idx", "type", "id").execute().addListener(new AssertingActionListener<>(DeleteAction.NAME, client.threadPool())); client.admin().cluster().prepareDeleteStoredScript("lang", "id").execute().addListener(new AssertingActionListener<>(DeleteStoredScriptAction.NAME, client.threadPool())); - client.prepareIndex("idx", "type", "id").setSource("source").execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool())); + client.prepareIndex("idx", "type", "id").setSource("source", XContentType.JSON).execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool())); // choosing arbitrary cluster admin actions to test client.admin().cluster().prepareClusterStats().execute().addListener(new AssertingActionListener<>(ClusterStatsAction.NAME, client.threadPool())); diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index a0a0025681779..c2f6c3b64faae 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -40,7 +41,8 @@ public void testDataNodes() throws Exception { internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build()); client().admin().indices().create(createIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet(); try { - client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet(); + client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON) + .timeout(timeValueSeconds(1))).actionGet(); fail("no allocation should happen"); } catch (UnavailableShardsException e) { // all is well @@ -51,7 +53,8 @@ public void testDataNodes() throws Exception { // still no shard should be allocated try { - client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet(); + client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON) + .timeout(timeValueSeconds(1))).actionGet(); fail("no allocation should happen"); } catch (UnavailableShardsException e) { // all is well @@ -61,7 +64,8 @@ public void testDataNodes() throws Exception { internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build()); assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).execute().actionGet().isTimedOut(), equalTo(false)); - IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet(); + IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); assertThat(indexResponse.getId(), equalTo("1")); assertThat(indexResponse.getType(), equalTo("type1")); } diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index 5709360de7441..e0642e726cae8 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.FaultDetection; import org.elasticsearch.discovery.zen.MembershipAction; @@ -527,7 +528,7 @@ public void testAckedIndexing() throws Exception { int shard = Math.floorMod(Murmur3HashFunction.hash(id), numPrimaries); logger.trace("[{}] indexing id [{}] through node [{}] targeting shard [{}]", name, id, node, shard); IndexResponse response = - client.prepareIndex("test", "type", id).setSource("{}").setTimeout(timeout).get(timeout); + client.prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).setTimeout(timeout).get(timeout); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); ackedDocs.put(id, node); logger.trace("[{}] indexed id [{}] through node [{}]", name, id, node); @@ -1119,7 +1120,8 @@ public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Excep final String node_2 = internalCluster().startDataOnlyNode(); List indexRequestBuilderList = new ArrayList<>(); for (int i = 0; i < 100; i++) { - indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc").setSource("{\"int_field\":1}")); + indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc") + .setSource("{\"int_field\":1}", XContentType.JSON)); } indexRandom(true, indexRequestBuilderList); SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_2, random()); diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java index e3556c8cc7c63..34b4bc05727e3 100644 --- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -185,7 +186,7 @@ public void testBulk() throws Exception { .add(client().prepareIndex().setIndex("test").setType("type1").setId("2").setSource(source("2", "test")).setCreate(true)) .add(client().prepareIndex().setIndex("test").setType("type1").setSource(source("3", "test"))) .add(client().prepareDelete().setIndex("test").setType("type1").setId("1")) - .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }")) // failure + .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }", XContentType.JSON)) // failure .execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java index 814a861139e0a..84166bb3f962e 100644 --- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java +++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -45,7 +46,7 @@ public class ShardInfoIT extends ESIntegTestCase { public void testIndexAndDelete() throws Exception { prepareIndex(1); - IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get(); + IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON).get(); assertShardInfo(indexResponse); DeleteResponse deleteResponse = client().prepareDelete("idx", "type", indexResponse.getId()).get(); assertShardInfo(deleteResponse); @@ -53,7 +54,8 @@ public void testIndexAndDelete() throws Exception { public void testUpdate() throws Exception { prepareIndex(1); - UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}").setDocAsUpsert(true).get(); + UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true) + .get(); assertShardInfo(updateResponse); } @@ -61,7 +63,7 @@ public void testBulkWithIndexAndDeleteItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { - bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}")); + bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); } BulkResponse bulkResponse = bulkRequestBuilder.get(); @@ -83,7 +85,8 @@ public void testBulkWithUpdateItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { - bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}").setDocAsUpsert(true)); + bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}", XContentType.JSON) + .setDocAsUpsert(true)); } BulkResponse bulkResponse = bulkRequestBuilder.get(); diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index f6164d2f3448e..577087ba9522b 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.test.ESIntegTestCase; @@ -629,8 +630,8 @@ public void testGetFieldsComplexField() throws Exception { logger.info("indexing documents"); - client().prepareIndex("my-index", "my-type1", "1").setSource(source).get(); - client().prepareIndex("my-index", "my-type2", "1").setSource(source).get(); + client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get(); + client().prepareIndex("my-index", "my-type2", "1").setSource(source, XContentType.JSON).get(); logger.info("checking real time retrieval"); @@ -698,7 +699,7 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException { " }\n" + " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"suggest\": {\n" + @@ -737,10 +738,10 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { " }\n" + " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); - client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").get(); + client().prepareIndex("test", "doc").setId("1").setSource("{}", XContentType.JSON).setParent("1").get(); String[] fieldsList = {"_parent"}; // before refresh - document is only in translog @@ -761,12 +762,12 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"text\": \"some text.\"\n" + "}\n"; - client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); + client().prepareIndex("test", "doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get(); String[] fieldsList = {"_routing"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); @@ -828,7 +829,7 @@ void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolea " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"text1\": \"some text.\"\n," + @@ -890,7 +891,7 @@ void indexSingleDocumentWithNumericFieldsGeneratedFromText(boolean stored, boole " }\n" + "}"; - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON)); ensureGreen(); String doc = "{\n" + " \"token_count\": \"A text with five words.\",\n" + diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java index 84a9fe2970bd0..7fa769b2613eb 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -40,10 +41,11 @@ public void testSetSource() throws InterruptedException, ExecutionException { map.put("test_field", "foobar"); IndexRequestBuilder[] builders = new IndexRequestBuilder[] { client().prepareIndex("test", "test").setSource((Object)"test_field", (Object)"foobar"), - client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))), + client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}", XContentType.JSON), + client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON), + client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON), + client().prepareIndex("test", "test") + .setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}")), XContentType.JSON), client().prepareIndex("test", "test").setSource(map) }; indexRandom(true, builders); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 28d67a4a1e2b6..80e453d665e3d 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.shard.IndexShard; @@ -232,7 +233,7 @@ public void testRefreshActuallyWorks() throws Exception { // now disable IndexMetaData metaData = IndexMetaData.builder(indexService.getMetaData()).settings(Settings.builder().put(indexService.getMetaData().getSettings()).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)).build(); indexService.updateMetaData(metaData); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 10); @@ -259,7 +260,7 @@ public void testAsyncFsyncActuallyWorks() throws Exception { IndexService indexService = createIndex("test", settings); ensureGreen("test"); assertTrue(indexService.getRefreshTask().mustReschedule()); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); assertBusy(() -> { assertFalse(shard.getTranslog().syncNeeded()); @@ -278,7 +279,7 @@ public void testRescheduleAsyncFsync() throws Exception { indexService.updateMetaData(metaData); assertNotNull(indexService.getFsyncTask()); assertTrue(indexService.getRefreshTask().mustReschedule()); - client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); IndexShard shard = indexService.getShard(0); assertBusy(() -> { assertFalse(shard.getTranslog().syncNeeded()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 3c9a1c16d16b5..f6fd7bbd0784a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -91,7 +92,7 @@ public void testTypes() throws Exception { public void testIndexIntoDefaultMapping() throws Throwable { // 1. test implicit index creation ExecutionException e = expectThrows(ExecutionException.class, () -> { - client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}").execute().get(); + client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}", XContentType.JSON).execute().get(); }); Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause()); if (throwable instanceof IllegalArgumentException) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java index ed0822d4d5391..67d7873b7b466 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java @@ -21,20 +21,16 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 3216d30bdaa8f..1bf876a5ef01d 100644 --- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.Uid; @@ -116,7 +117,7 @@ protected class ReplicationGroup implements AutoCloseable, Iterable public int indexDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet())) - .source("{}"); + .source("{}", XContentType.JSON); final IndexResponse response = index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); } @@ -126,7 +127,7 @@ public int indexDocs(final int numOfDoc) throws Exception { public int appendDocs(final int numOfDoc) throws Exception { for (int doc = 0; doc < numOfDoc; doc++) { - final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}"); + final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON); final IndexResponse response = index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); } diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index 41edc303f9dbd..39d8778c2a4b8 100644 --- a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.index.engine.InternalEngineTests; @@ -110,7 +111,7 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa public void testInheritMaxValidAutoIDTimestampOnRecovery() throws Exception { try (ReplicationGroup shards = createGroup(0)) { shards.startAll(); - final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}"); + final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON); indexRequest.onRetry(); // force an update of the timestamp final IndexResponse response = shards.index(indexRequest); assertEquals(DocWriteResponse.Result.CREATED, response.getResult()); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 3e35ed357fff8..7af2affc8150c 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; @@ -146,7 +147,7 @@ public void testLockTryingToDelete() throws Exception { public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0); @@ -163,14 +164,14 @@ public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { public void testDurableFlagHasEffect() { createIndex("test"); ensureGreen(); - client().prepareIndex("test", "bar", "1").setSource("{}").get(); + client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).get(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); setDurability(shard, Translog.Durability.REQUEST); assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.ASYNC); - client().prepareIndex("test", "bar", "2").setSource("{}").get(); + client().prepareIndex("test", "bar", "2").setSource("{}", XContentType.JSON).get(); assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); client().prepareDelete("test", "bar", "1").get(); @@ -181,13 +182,13 @@ public void testDurableFlagHasEffect() { assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "3").setSource("{}")) + .add(client().prepareIndex("test", "bar", "3").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "bar", "1")).get()); assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.ASYNC); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "4").setSource("{}")) + .add(client().prepareIndex("test", "bar", "4").setSource("{}", XContentType.JSON)) .add(client().prepareDelete("test", "bar", "3")).get()); setDurability(shard, Translog.Durability.REQUEST); assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded()); @@ -218,7 +219,7 @@ public void testIndexDirIsDeletedWhenShardRemoved() throws Exception { .build(); createIndex("test", idxSettings); ensureGreen("test"); - client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); SearchResponse response = client().prepareSearch("test").get(); assertHitCount(response, 1L); client().admin().indices().prepareDelete("test").get(); @@ -230,7 +231,7 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); for (int i = 0; i < 50; i++) { - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); } ensureGreen("test"); InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); @@ -266,7 +267,7 @@ public void testIndexCanChangeCustomDataPath() throws Exception { logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString()); createIndex(INDEX, sb); ensureGreen(INDEX); - client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex(INDEX, "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get(); assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L)); @@ -323,7 +324,8 @@ public void testMaybeFlush() throws Exception { client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(117 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); - client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "0") + .setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldFlush()); ParsedDocument doc = testParsedDocument( "1", @@ -336,7 +338,8 @@ public void testMaybeFlush() throws Exception { shard.index(index); assertTrue(shard.shouldFlush()); assertEquals(2, shard.getEngine().getTranslog().totalOperations()); - client().prepareIndex("test", "test", "2").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON) + .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertBusy(() -> { // this is async assertFalse(shard.shouldFlush()); }); @@ -369,7 +372,8 @@ public void testStressMaybeFlush() throws Exception { client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put( IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(117/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); - client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); + client().prepareIndex("test", "test", "0").setSource("{}", XContentType.JSON) + .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldFlush()); final AtomicBoolean running = new AtomicBoolean(true); final int numThreads = randomIntBetween(2, 4); @@ -394,7 +398,7 @@ public void run() { barrier.await(); FlushStats flushStats = shard.flushStats(); long total = flushStats.getTotal(); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); assertBusy(() -> assertEquals(total + 1, shard.flushStats().getTotal())); running.set(false); for (int i = 0; i < threads.length; i++) { @@ -409,9 +413,9 @@ public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexService(resolveIndex("test")); IndexShard shard = indexService.getShardOrNull(0); - client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get(); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); client().prepareDelete("test", "test", "0").get(); - client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {}; shard.close("simon says", false); diff --git a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index 82482b1f821f0..fe4ab2e363a88 100644 --- a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -57,9 +58,9 @@ public void testIndexNameDateMathExpressions() { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get(); - client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get(); - client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get(); + client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); @@ -116,9 +117,9 @@ public void testAutoCreateIndexWithDateMathExpression() throws Exception { String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; - client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get(); - client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get(); - client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get(); + client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get(); + client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index e9a717f663676..fee2964fe126e 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexSearcherWrapper; @@ -423,7 +424,7 @@ public void testTranslogRecoveryWorksWithIMC() throws IOException { IndexService indexService = indicesService.indexService(resolveIndex("test")); IndexShard shard = indexService.getShardOrNull(0); for (int i = 0; i < 100; i++) { - client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}").get(); + client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); } IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {}; diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index d974ea348cfac..50979cf611d07 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; @@ -57,7 +58,7 @@ public void testWaitIfOngoing() throws InterruptedException { final int numIters = scaledRandomIntBetween(10, 30); for (int i = 0; i < numIters; i++) { for (int j = 0; j < 10; j++) { - client().prepareIndex("test", "test").setSource("{}").get(); + client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get(); } final CountDownLatch latch = new CountDownLatch(10); final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); @@ -164,7 +165,7 @@ public void testSyncedFlushWithConcurrentIndexing() throws Exception { @Override public void run() { while (stop.get() == false) { - client().prepareIndex().setIndex("test").setType("doc").setSource("{}").get(); + client().prepareIndex().setIndex("test").setType("doc").setSource("{}", XContentType.JSON).get(); numDocs.incrementAndGet(); } } diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java index c35df81cfd673..59784456e988b 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; @@ -42,7 +43,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { public void testModificationPreventsFlushing() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -54,7 +55,7 @@ public void testModificationPreventsFlushing() throws InterruptedException { assertEquals("exactly one active shard", 1, activeShards.size()); Map commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId); assertEquals("exactly one commit id", 1, commitIds.size()); - client().prepareIndex("test", "test", "2").setSource("{}").get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get(); String syncId = UUIDs.base64UUID(); SyncedFlushUtil.LatchedListener listener = new SyncedFlushUtil.LatchedListener<>(); flushService.sendSyncRequests(syncId, activeShards, state, commitIds, shardId, shardRoutingTable.size(), listener); @@ -86,7 +87,7 @@ public void testModificationPreventsFlushing() throws InterruptedException { public void testSingleShardSuccess() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -106,7 +107,7 @@ public void testSingleShardSuccess() throws InterruptedException { public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -161,7 +162,7 @@ public void testSyncFailsOnIndexClosedOrMissing() throws InterruptedException { public void testFailAfterIntermediateCommit() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); @@ -174,7 +175,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException { Map commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId); assertEquals("exactly one commit id", 1, commitIds.size()); if (randomBoolean()) { - client().prepareIndex("test", "test", "2").setSource("{}").get(); + client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get(); } client().admin().indices().prepareFlush("test").setForce(true).get(); String syncId = UUIDs.base64UUID(); @@ -194,7 +195,7 @@ public void testFailAfterIntermediateCommit() throws InterruptedException { public void testFailWhenCommitIsMissing() throws InterruptedException { createIndex("test"); - client().prepareIndex("test", "test", "1").setSource("{}").get(); + client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 63517dbdc97af..aa960594dedc3 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.CircuitBreakerStats; @@ -174,7 +175,7 @@ public void testRamAccountingTermsEnum() throws Exception { // Create an index where the mappings have a field data filter assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " + - "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}")); + "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", XContentType.JSON)); ensureGreen("ramtest"); diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 424edb42e68d1..72a789ddc1c1d 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.store.Store; @@ -576,7 +577,7 @@ public void testDisconnectsWhileRecovering() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); ensureSearchable(indexName); @@ -686,7 +687,7 @@ public void testDisconnectsDuringRecovery() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); ensureSearchable(indexName); diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 519df00c06c69..91673630c3d49 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; @@ -739,8 +740,8 @@ public void testFieldDataFieldsParam() throws Exception { ensureGreen(); - client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); - client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); refresh(); client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet(); @@ -786,8 +787,8 @@ public void testCompletionFieldsParam() throws Exception { "{ \"properties\": { \"bar\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}")); ensureGreen(); - client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); - client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); refresh(); IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); @@ -1125,7 +1126,7 @@ public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierExcepti final IndexResponse response = client() .prepareIndex("test", "type", id) - .setSource("{}") + .setSource("{}", XContentType.JSON) .get(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); } diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 6469b660f0b98..e5e5f7c456b15 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.InvalidAliasNameException; @@ -335,7 +336,7 @@ public void testBrokenMapping() throws Exception { MapperParsingException e = expectThrows( MapperParsingException.class, () -> client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "abcde") + .addMapping("type1", "abcde", XContentType.JSON) .get()); assertThat(e.getMessage(), containsString("Failed to parse mapping ")); @@ -372,7 +373,7 @@ public void testIndexTemplateWithAliases() throws Exception { client().admin().indices().preparePutTemplate("template_with_aliases") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}") + .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON) .addAlias(new Alias("simple_alias")) .addAlias(new Alias("templated_alias-{index}")) .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}")) @@ -434,7 +435,7 @@ public void testIndexTemplateWithAliasesInSource() { " }\n" + " }\n" + " }\n" + - "}").get(); + "}", XContentType.JSON).get(); assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2")); @@ -635,8 +636,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio .setOrder(0) .addAlias(new Alias("alias4").filter(termQuery("field", "value"))).get(); - client().prepareIndex("a1", "test", "test").setSource("{}").get(); - BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get(); + client().prepareIndex("a1", "test", "test").setSource("{}", XContentType.JSON).get(); + BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("a2")); @@ -644,8 +645,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio assertThat(response.getItems()[0].getId(), equalTo("test")); assertThat(response.getItems()[0].getVersion(), equalTo(1L)); - client().prepareIndex("b1", "test", "test").setSource("{}").get(); - response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get(); + client().prepareIndex("b1", "test", "test").setSource("{}", XContentType.JSON).get(); + response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("b2")); @@ -653,8 +654,8 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio assertThat(response.getItems()[0].getId(), equalTo("test")); assertThat(response.getItems()[0].getVersion(), equalTo(1L)); - client().prepareIndex("c1", "test", "test").setSource("{}").get(); - response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get(); + client().prepareIndex("c1", "test", "test").setSource("{}", XContentType.JSON).get(); + response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getIndex(), equalTo("c2")); @@ -670,9 +671,9 @@ public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exceptio // So the aliases defined in the index template for this index will not fail // even though the fields in the alias fields don't exist yet and indexing into // an index that doesn't exist yet will succeed - client().prepareIndex("d1", "test", "test").setSource("{}").get(); + client().prepareIndex("d1", "test", "test").setSource("{}", XContentType.JSON).get(); - response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get(); + response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}", XContentType.JSON)).get(); assertThat(response.hasFailures(), is(false)); assertThat(response.getItems()[0].isFailed(), equalTo(false)); assertThat(response.getItems()[0].getId(), equalTo("test")); @@ -702,7 +703,7 @@ public void testCombineTemplates() throws Exception{ " }\n" + " }\n" + " }\n" + - " }\n") + " }\n", XContentType.JSON) .get(); // put template using custom_1 analyzer diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 14335bde076de..1882031f3b90c 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -38,6 +38,7 @@ import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -81,7 +82,7 @@ public void testSimulate() throws Exception { .endObject() .endArray() .endObject().bytes(); - client().admin().cluster().preparePutPipeline("_id", pipelineSource) + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON) .get(); GetPipelineResponse getResponse = client().admin().cluster().prepareGetPipeline("_id") .get(); @@ -104,10 +105,10 @@ public void testSimulate() throws Exception { .endObject().bytes(); SimulatePipelineResponse response; if (randomBoolean()) { - response = client().admin().cluster().prepareSimulatePipeline(bytes) + response = client().admin().cluster().prepareSimulatePipeline(bytes, XContentType.JSON) .setId("_id").get(); } else { - SimulatePipelineRequest request = new SimulatePipelineRequest(bytes); + SimulatePipelineRequest request = new SimulatePipelineRequest(bytes, XContentType.JSON); request.setId("_id"); response = client().admin().cluster().simulatePipeline(request).get(); } @@ -137,7 +138,7 @@ public void testBulkWithIngestFailures() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); client().admin().cluster().putPipeline(putPipelineRequest).get(); int numRequests = scaledRandomIntBetween(32, 128); @@ -177,7 +178,7 @@ public void test() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); client().admin().cluster().putPipeline(putPipelineRequest).get(); GetPipelineRequest getPipelineRequest = new GetPipelineRequest("_id"); @@ -219,7 +220,7 @@ public void testPutWithPipelineFactoryError() throws Exception { .endObject() .endArray() .endObject().bytes(); - PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); try { client().admin().cluster().putPipeline(putPipelineRequest).get(); } catch (ExecutionException e) { diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java index e2fca8b4112a9..518b775d7f802 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java @@ -42,10 +42,10 @@ public class IngestMetadataTests extends ESTestCase { public void testFromXContent() throws IOException { PipelineConfiguration pipeline = new PipelineConfiguration( - "1",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "1", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); PipelineConfiguration pipeline2 = new PipelineConfiguration( - "2",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}") + "2", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}"), XContentType.JSON ); Map map = new HashMap<>(); map.put(pipeline.getId(), pipeline); @@ -72,14 +72,14 @@ public void testDiff() throws Exception { BytesReference pipelineConfig = new BytesArray("{}"); Map pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", pipelineConfig)); - pipelines.put("2", new PipelineConfiguration("2", pipelineConfig)); + pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", pipelineConfig, XContentType.JSON)); IngestMetadata ingestMetadata1 = new IngestMetadata(pipelines); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", pipelineConfig)); - pipelines.put("3", new PipelineConfiguration("3", pipelineConfig)); - pipelines.put("4", new PipelineConfiguration("4", pipelineConfig)); + pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)); + pipelines.put("3", new PipelineConfiguration("3", pipelineConfig, XContentType.JSON)); + pipelines.put("4", new PipelineConfiguration("4", pipelineConfig, XContentType.JSON)); IngestMetadata ingestMetadata2 = new IngestMetadata(pipelines); IngestMetadata.IngestMetadataDiff diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata2.diff(ingestMetadata1); @@ -92,13 +92,13 @@ public void testDiff() throws Exception { IngestMetadata endResult = (IngestMetadata) diff.apply(ingestMetadata2); assertThat(endResult, not(equalTo(ingestMetadata1))); assertThat(endResult.getPipelines().size(), equalTo(3)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig))); - assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig, XContentType.JSON))); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"))); - pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"))); + pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"), XContentType.JSON)); IngestMetadata ingestMetadata3 = new IngestMetadata(pipelines); diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata3.diff(ingestMetadata1); @@ -108,12 +108,12 @@ public void testDiff() throws Exception { endResult = (IngestMetadata) diff.apply(ingestMetadata3); assertThat(endResult, equalTo(ingestMetadata1)); assertThat(endResult.getPipelines().size(), equalTo(2)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig, XContentType.JSON))); pipelines = new HashMap<>(); - pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"))); - pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"))); + pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON)); + pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON)); IngestMetadata ingestMetadata4 = new IngestMetadata(pipelines); diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata4.diff(ingestMetadata1); @@ -123,7 +123,8 @@ public void testDiff() throws Exception { endResult = (IngestMetadata) diff.apply(ingestMetadata4); assertThat(endResult, not(equalTo(ingestMetadata1))); assertThat(endResult.getPipelines().size(), equalTo(2)); - assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig))); - assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}")))); + assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON))); + assertThat(endResult.getPipelines().get("2"), + equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON))); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index 83072636b762b..1a8e6e873e01d 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.node.NodeService; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -69,7 +70,7 @@ public void testFailPipelineCreation() throws Exception { ensureStableCluster(2, node2); try { - client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); fail("exception expected"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("Processor type [test] is not installed on node")); @@ -82,7 +83,7 @@ public void testFailPipelineCreationProcessorNotInstalledOnMasterNode() throws E internalCluster().startNode(); try { - client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("No processor type exists with name [test]")); @@ -95,7 +96,7 @@ public void testFailStartNode() throws Exception { installPlugin = true; String node1 = internalCluster().startNode(); - WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource).get(); + WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get(); assertThat(response.isAcknowledged(), is(true)); Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id"); assertThat(pipeline, notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java new file mode 100644 index 0000000000000..6aaac41b8beab --- /dev/null +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class PipelineConfigurationTests extends ESTestCase { + + public void testSerialization() throws IOException { + PipelineConfiguration configuration = new PipelineConfiguration("1", + new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); + assertEquals(XContentType.YAML, configuration.getXContentType()); + + BytesStreamOutput out = new BytesStreamOutput(); + configuration.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + PipelineConfiguration serialized = PipelineConfiguration.readFrom(in); + assertEquals(XContentType.YAML, serialized.getXContentType()); + assertEquals("{}", serialized.getConfig().utf8ToString()); + + out = new BytesStreamOutput(); + out.setVersion(Version.V_5_0_0); + configuration.writeTo(out); + in = StreamInput.wrap(out.bytes().toBytesRef().bytes); + in.setVersion(Version.V_5_0_0); + + serialized = PipelineConfiguration.readFrom(in); + assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals("{}", serialized.getConfig().utf8ToString()); + } + + public void testParser() throws IOException { + ContextParser parser = PipelineConfiguration.getParser(); + XContentType xContentType = randomFrom(XContentType.values()); + final BytesReference bytes; + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + new PipelineConfiguration("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON) + .toXContent(builder, ToXContent.EMPTY_PARAMS); + bytes = builder.bytes(); + } + + XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes); + PipelineConfiguration parsed = parser.parse(xContentParser, null); + assertEquals(xContentType, parsed.getXContentType()); + assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); + assertEquals("1", parsed.getId()); + } +} diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java index c00e9254ab289..3d409c61d5964 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.CustomTypeSafeMatcher; @@ -331,8 +332,8 @@ public void testStats() throws Exception { when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, null, new CompoundProcessor(mock(Processor.class)))); Map configurationMap = new HashMap<>(); - configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); - configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"))); + configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON)); + configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); @SuppressWarnings("unchecked") @@ -361,14 +362,14 @@ public void testStats() throws Exception { // issue: https://github.com/elastic/elasticsearch/issues/18126 public void testUpdatingStatsWhenRemovingPipelineWorks() throws Exception { Map configurationMap = new HashMap<>(); - configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); - configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"))); + configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON)); + configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id1")); assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id2")); configurationMap = new HashMap<>(); - configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"))); + configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"), XContentType.JSON)); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id1"))); assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id2"))); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java index 5967354710c7b..e91bb7e2e6e17 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -102,7 +103,7 @@ public void testUpdatePipelines() { assertThat(store.pipelines.size(), is(0)); PipelineConfiguration pipeline = new PipelineConfiguration( - "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", pipeline)); clusterState = ClusterState.builder(clusterState) @@ -123,7 +124,7 @@ public void testPut() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // add a new pipeline: - PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}")); + PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -134,7 +135,8 @@ public void testPut() { assertThat(pipeline.getProcessors().size(), equalTo(0)); // overwrite existing pipeline: - putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}")); + putRequest = + new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"), XContentType.JSON); previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -151,7 +153,8 @@ public void testPutWithErrorResponse() { assertThat(pipeline, nullValue()); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); - PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}")); + PutPipelineRequest putRequest = + new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); try { @@ -166,7 +169,7 @@ public void testPutWithErrorResponse() { public void testDelete() { PipelineConfiguration config = new PipelineConfiguration( - "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}") + "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON ); IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config)); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); @@ -197,9 +200,9 @@ public void testDeleteUsingWildcard() { BytesArray definition = new BytesArray( "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}" ); - pipelines.put("p1", new PipelineConfiguration("p1", definition)); - pipelines.put("p2", new PipelineConfiguration("p2", definition)); - pipelines.put("q1", new PipelineConfiguration("q1", definition)); + pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON)); + pipelines.put("p2", new PipelineConfiguration("p2", definition, XContentType.JSON)); + pipelines.put("q1", new PipelineConfiguration("q1", definition, XContentType.JSON)); IngestMetadata ingestMetadata = new IngestMetadata(pipelines); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); ClusterState previousClusterState = clusterState; @@ -245,7 +248,7 @@ public void testDeleteWithExistingUnmatchedPipelines() { BytesArray definition = new BytesArray( "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}" ); - pipelines.put("p1", new PipelineConfiguration("p1", definition)); + pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON)); IngestMetadata ingestMetadata = new IngestMetadata(pipelines); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); ClusterState previousClusterState = clusterState; @@ -266,10 +269,10 @@ public void testDeleteWithExistingUnmatchedPipelines() { public void testGetPipelines() { Map configs = new HashMap<>(); configs.put("_id1", new PipelineConfiguration( - "_id1", new BytesArray("{\"processors\": []}") + "_id1", new BytesArray("{\"processors\": []}"), XContentType.JSON )); configs.put("_id2", new PipelineConfiguration( - "_id2", new BytesArray("{\"processors\": []}") + "_id2", new BytesArray("{\"processors\": []}"), XContentType.JSON )); assertThat(store.innerGetPipelines(null, "_id1").isEmpty(), is(true)); @@ -311,7 +314,7 @@ public void testCrud() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty PutPipelineRequest putRequest = new PutPipelineRequest(id, - new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")); + new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON); ClusterState previousClusterState = clusterState; clusterState = store.innerPut(putRequest, clusterState); store.innerUpdatePipelines(previousClusterState, clusterState); @@ -332,7 +335,8 @@ public void testCrud() throws Exception { public void testValidate() throws Exception { PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray( - "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}")); + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"), + XContentType.JSON); DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -355,7 +359,7 @@ public void testValidate() throws Exception { public void testValidateNoIngestInfo() throws Exception { PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray( - "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")); + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON); try { store.validatePipeline(Collections.emptyMap(), putRequest); fail("exception expected"); diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java index 5c499eae4d8a9..3f0eac7762f0b 100644 --- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java +++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.test.ESIntegTestCase; @@ -143,7 +144,7 @@ public void testThatSourceFilteringIsSupported() throws Exception { .field("excluded", "should not be seen") .endObject().bytes(); for (int i = 0; i < 100; i++) { - client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef).get(); + client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef, XContentType.JSON).get(); } MultiGetRequestBuilder request = client().prepareMultiGet(); diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index bbd1ad56bd0b3..a1d27a84fea56 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbersService; @@ -334,12 +335,12 @@ public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardSt List builders1 = new ArrayList<>(); for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) { - builders1.add(client().prepareIndex("test", "type").setSource("{}")); + builders1.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON)); } List builders2 = new ArrayList<>(); for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) { - builders2.add(client().prepareIndex("test", "type").setSource("{}")); + builders2.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON)); } logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]); @@ -392,7 +393,7 @@ public void testCancellationCleansTempFiles() throws Exception { List requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { - requests.add(client().prepareIndex(indexName, "type").setSource("{}")); + requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON)); } indexRandom(true, requests); assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").setWaitForGreenStatus().get().isTimedOut()); diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java index e860cfc9ebd54..bf8bce8ae6334 100644 --- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.client.Requests.flushRequest; @@ -52,12 +53,12 @@ public void testSimpleRecovery() throws Exception { NumShards numShards = getNumShards("test"); - client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet(); + client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet(); FlushResponse flushResponse = client().admin().indices().flush(flushRequest("test")).actionGet(); assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(flushResponse.getFailedShards(), equalTo(0)); - client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"))).actionGet(); + client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"), XContentType.JSON)).actionGet(); RefreshResponse refreshResponse = client().admin().indices().refresh(refreshRequest("test")).actionGet(); assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 6e3d0025eb8ae..ad93eab3b7ba4 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -35,7 +35,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collections; -import java.util.Map; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -158,7 +157,7 @@ public void testConvert() throws IOException { public void testResponseWhenPathContainsEncodingError() throws IOException { final String path = "%a"; - final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path) { + final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, Collections.emptyMap()) { @Override public Method method() { return null; @@ -178,16 +177,6 @@ public boolean hasContent() { public BytesReference content() { return null; } - - @Override - public String header(String name) { - return null; - } - - @Override - public Iterable> headers() { - return null; - } }; final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath())); final RestChannel channel = new DetailedExceptionRestChannel(request); diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index e7064554908f3..46cd40852b4c4 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -19,10 +19,12 @@ package org.elasticsearch.rest; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,10 +43,10 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.http.HttpInfo; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.HttpStats; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -98,10 +100,10 @@ public void testApplyRelevantHeaders() throws Exception { assertNull(threadContext.getHeader("header.3")); }); threadContext.putHeader("header.3", "true"); - Map restHeaders = new HashMap<>(); - restHeaders.put("header.1", "true"); - restHeaders.put("header.2", "true"); - restHeaders.put("header.3", "false"); + Map> restHeaders = new HashMap<>(); + restHeaders.put("header.1", Collections.singletonList("true")); + restHeaders.put("header.2", Collections.singletonList("true")); + restHeaders.put("header.3", Collections.singletonList("false")); restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null, threadContext); assertNull(threadContext.getHeader("header.1")); @@ -203,7 +205,7 @@ public boolean canTripCircuitBreaker() { public void testDispatchRequestAddsAndFreesBytesOnSuccess() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/", content); + TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -215,7 +217,7 @@ public void testDispatchRequestAddsAndFreesBytesOnSuccess() { public void testDispatchRequestAddsAndFreesBytesOnError() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/error", content); + TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -228,7 +230,7 @@ public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() { int contentLength = BREAKER_LIMIT.bytesAsInt(); String content = randomAsciiOfLength(contentLength); // we will produce an error in the rest handler and one more when sending the error response - TestRestRequest request = new TestRestRequest("/error", content); + TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON); ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -240,7 +242,7 @@ public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() { public void testDispatchRequestLimitsBytes() { int contentLength = BREAKER_LIMIT.bytesAsInt() + 1; String content = randomAsciiOfLength(contentLength); - TestRestRequest request = new TestRestRequest("/", content); + TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -249,6 +251,37 @@ public void testDispatchRequestLimitsBytes() { assertEquals(0, inFlightRequestsBreaker.getUsed()); } + public void testDispatchRequiresContentTypeForRequestsWithContent() { + String content = randomAsciiOfLengthBetween(1, 32); + TestRestRequest request = new TestRestRequest("/", content, null); + AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + } + + public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() { + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + } + + public void testDispatchWorksWithPlainText() { + String content = randomAsciiOfLengthBetween(1, 32); + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withContent(new BytesArray(content.getBytes(StandardCharsets.UTF_8)), null).withPath("/") + .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + } + private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { @@ -287,6 +320,7 @@ public HttpStats stats() { private static final class AssertingChannel extends AbstractRestChannel { private final RestStatus expectedStatus; + private final AtomicBoolean sendResponseCalled = new AtomicBoolean(false); protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) { super(request, detailedErrorsEnabled); @@ -296,6 +330,7 @@ protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, R @Override public void sendResponse(RestResponse response) { assertEquals(expectedStatus, response.status()); + sendResponseCalled.set(true); } } @@ -315,8 +350,9 @@ private static final class TestRestRequest extends RestRequest { private final BytesReference content; - private TestRestRequest(String path, String content) { - super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path); + private TestRestRequest(String path, String content, XContentType xContentType) { + super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, xContentType == null ? + Collections.emptyMap() : Collections.singletonMap("Content-Type", Collections.singletonList(xContentType.mediaType()))); this.content = new BytesArray(content); } @@ -340,15 +376,5 @@ public BytesReference content() { return content; } - @Override - public String header(String name) { - return null; - } - - @Override - public Iterable> headers() { - return null; - } - } } diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java index 7f0184e20c60b..d96c3212e3222 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -24,11 +24,15 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; @@ -54,10 +58,12 @@ public void testApplyContentParser() throws IOException { } public void testContentOrSourceParam() throws IOException { - assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam()); - assertEquals(new BytesArray("stuff"), new ContentRestRequest("", singletonMap("source", "stuff")).contentOrSourceParam()); + assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam().v2()); + assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam().v2()); + assertEquals(new BytesArray("stuff"), + new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam().v2()); + assertEquals(new BytesArray("{\"foo\": \"stuff\"}"), + new ContentRestRequest("", singletonMap("source", "{\"foo\": \"stuff\"}")).contentOrSourceParam().v2()); } public void testHasContentOrSourceParam() throws IOException { @@ -85,10 +91,59 @@ public void testWithContentOrSourceParamParserOrNull() throws IOException { assertEquals(emptyMap(), parser.map())); } + public void testContentTypeParsing() { + for (XContentType xContentType : XContentType.values()) { + Map> map = new HashMap<>(); + map.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + ContentRestRequest restRequest = new ContentRestRequest("", Collections.emptyMap(), map); + assertEquals(xContentType, restRequest.getXContentType()); + assertFalse(restRequest.isPlainText()); + + map = new HashMap<>(); + map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters())); + restRequest = new ContentRestRequest("", Collections.emptyMap(), map); + assertEquals(xContentType, restRequest.getXContentType()); + assertFalse(restRequest.isPlainText()); + } + } + + public void testPlainTextSupport() { + ContentRestRequest restRequest = new ContentRestRequest(randomAsciiOfLengthBetween(1, 30), Collections.emptyMap(), + Collections.singletonMap("Content-Type", + Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8")))); + assertNull(restRequest.getXContentType()); + assertTrue(restRequest.isPlainText()); + } + + public void testMalformedContentTypeHeader() { + final String type = randomFrom("text", "text/:ain; charset=utf-8", "text/plain\";charset=utf-8", ":", "/", "t:/plain"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(), + Collections.singletonMap("Content-Type", Collections.singletonList(type)))); + assertEquals("invalid Content-Type header [" + type + "]", e.getMessage()); + } + + public void testNoContentTypeHeader() { + ContentRestRequest contentRestRequest = new ContentRestRequest("", Collections.emptyMap(), Collections.emptyMap()); + assertNull(contentRestRequest.getXContentType()); + assertFalse(contentRestRequest.isPlainText()); + } + + public void testMultipleContentTypeHeaders() { + List headers = new ArrayList<>(randomUnique(() -> randomAsciiOfLengthBetween(1, 16), randomIntBetween(2, 10))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(), + Collections.singletonMap("Content-Type", headers))); + assertEquals("only one Content-Type header should be provided", e.getMessage()); + } + private static final class ContentRestRequest extends RestRequest { private final BytesArray content; - public ContentRestRequest(String content, Map params) { - super(NamedXContentRegistry.EMPTY, params, "not used by this test"); + + ContentRestRequest(String content, Map params) { + this(content, params, Collections.singletonMap("Content-Type", Collections.singletonList("application/json"))); + } + + ContentRestRequest(String content, Map params, Map> headers) { + super(NamedXContentRegistry.EMPTY, params, "not used by this test", headers); this.content = new BytesArray(content); } @@ -96,7 +151,7 @@ public ContentRestRequest(String content, Map params) { public boolean hasContent() { return Strings.hasLength(content); } - + @Override public BytesReference content() { return content; @@ -111,15 +166,5 @@ public String uri() { public Method method() { throw new UnsupportedOperationException("Not used by this test"); } - - @Override - public Iterable> headers() { - throw new UnsupportedOperationException("Not used by this test"); - } - - @Override - public String header(String name) { - throw new UnsupportedOperationException("Not used by this test"); - } } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java index da52358d3c3c2..958b9e5222fad 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; @@ -92,7 +93,8 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception { RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java index 38ed76cae5249..4f7bc1bedd9d6 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java @@ -104,35 +104,35 @@ public void testThatDisplayHeadersAreNotAddedTwice() throws Exception { } public void testThatWeUseTheAcceptHeaderJson() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_JSON), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY); } public void testThatWeUseTheAcceptHeaderYaml() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_YAML), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY); } public void testThatWeUseTheAcceptHeaderSmile() throws Exception { - assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_SMILE), + assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE); } public void testThatWeUseTheAcceptHeaderCbor() throws Exception { - assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_CBOR), + assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_CBOR)), APPLICATION_CBOR); } public void testThatWeUseTheAcceptHeaderText() throws Exception { - assertResponse(Collections.singletonMap(ACCEPT, TEXT_PLAIN), + assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY); } public void testIgnoreContentType() throws Exception { - assertResponse(Collections.singletonMap(CONTENT_TYPE, APPLICATION_JSON), + assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY); } @@ -252,7 +252,7 @@ public void testMultiSort() { assertEquals(Arrays.asList(1,0,2), rowOrder); } - private RestResponse assertResponseContentType(Map headers, String mediaType) throws Exception { + private RestResponse assertResponseContentType(Map> headers, String mediaType) throws Exception { FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build(); table.startRow(); table.addCell("foo"); @@ -274,7 +274,7 @@ public void sendResponse(RestResponse response) { return response; } - private void assertResponse(Map headers, String mediaType, String body) throws Exception { + private void assertResponse(Map> headers, String mediaType, String body) throws Exception { RestResponse response = assertResponseContentType(headers, mediaType); assertThat(response.content().utf8ToString(), equalTo(body)); } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java index 4b2b9a49dc032..58b372a4f5a75 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java @@ -42,23 +42,23 @@ public void testGetScript() throws Exception { XContentBuilder sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().startObject("template").field("field", "value").endObject().endObject(); - builder.storeScript("lang", "template", sourceBuilder.bytes()); + builder.storeScript("lang", "template", sourceBuilder.bytes(), sourceBuilder.contentType()); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("template", "value").endObject(); - builder.storeScript("lang", "template_field", sourceBuilder.bytes()); + builder.storeScript("lang", "template_field", sourceBuilder.bytes(), sourceBuilder.contentType()); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().startObject("script").field("field", "value").endObject().endObject(); - builder.storeScript("lang", "script", sourceBuilder.bytes()); + builder.storeScript("lang", "script", sourceBuilder.bytes(), sourceBuilder.contentType()); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("script", "value").endObject(); - builder.storeScript("lang", "script_field", sourceBuilder.bytes()); + builder.storeScript("lang", "script_field", sourceBuilder.bytes(), sourceBuilder.contentType()); sourceBuilder = XContentFactory.jsonBuilder(); sourceBuilder.startObject().field("field", "value").endObject(); - builder.storeScript("lang", "any", sourceBuilder.bytes()); + builder.storeScript("lang", "any", sourceBuilder.bytes(), sourceBuilder.contentType()); ScriptMetaData scriptMetaData = builder.build(); assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "template")); @@ -97,15 +97,15 @@ public void testReadFromWriteTo() throws IOException { public void testDiff() throws Exception { ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null); - builder.storeScript("lang", "1", new BytesArray("{\"foo\":\"abc\"}")); - builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"def\"}")); - builder.storeScript("lang", "3", new BytesArray("{\"foo\":\"ghi\"}")); + builder.storeScript("lang", "1", new BytesArray("{\"foo\":\"abc\"}"), XContentType.JSON); + builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"def\"}"), XContentType.JSON); + builder.storeScript("lang", "3", new BytesArray("{\"foo\":\"ghi\"}"), XContentType.JSON); ScriptMetaData scriptMetaData1 = builder.build(); builder = new ScriptMetaData.Builder(scriptMetaData1); - builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"changed\"}")); + builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"changed\"}"), XContentType.JSON); builder.deleteScript("lang", "3"); - builder.storeScript("lang", "4", new BytesArray("{\"foo\":\"jkl\"}")); + builder.storeScript("lang", "4", new BytesArray("{\"foo\":\"jkl\"}"), XContentType.JSON); ScriptMetaData scriptMetaData2 = builder.build(); ScriptMetaData.ScriptMetadataDiff diff = (ScriptMetaData.ScriptMetadataDiff) scriptMetaData2.diff(scriptMetaData1); @@ -124,12 +124,13 @@ public void testDiff() throws Exception { public void testBuilder() { ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null); - builder.storeScript("_lang", "_id", new BytesArray("{\"script\":\"1 + 1\"}")); + builder.storeScript("_lang", "_id", new BytesArray("{\"script\":\"1 + 1\"}"), XContentType.JSON); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> builder.storeScript("_lang#", "_id", new BytesArray("{\"foo\": \"bar\"}"))); + () -> builder.storeScript("_lang#", "_id", new BytesArray("{\"foo\": \"bar\"}"), XContentType.JSON)); assertEquals("stored script language can't contain: '#'", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> builder.storeScript("_lang", "_id#", new BytesArray("{\"foo\": \"bar\"}"))); + e = expectThrows(IllegalArgumentException.class, + () -> builder.storeScript("_lang", "_id#", new BytesArray("{\"foo\": \"bar\"}"), XContentType.JSON)); assertEquals("stored script id can't contain: '#'", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> builder.deleteScript("_lang#", "_id")); assertEquals("stored script language can't contain: '#'", e.getMessage()); @@ -147,7 +148,7 @@ private ScriptMetaData randomScriptMetaData(XContentType sourceContentType) thro String lang = randomAsciiOfLength(4); XContentBuilder sourceBuilder = XContentBuilder.builder(sourceContentType.xContent()); sourceBuilder.startObject().field(randomAsciiOfLength(4), randomAsciiOfLength(4)).endObject(); - builder.storeScript(lang, randomAsciiOfLength(i + 1), sourceBuilder.bytes()); + builder.storeScript(lang, randomAsciiOfLength(i + 1), sourceBuilder.bytes(), sourceBuilder.contentType()); } return builder.build(); } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 9d0c5b1867f23..769f407d4c1fa 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; @@ -435,7 +436,7 @@ public void testDeleteScript() throws Exception { .metaData(MetaData.builder() .putCustom(ScriptMetaData.TYPE, new ScriptMetaData.Builder(null).storeScript("_lang", "_id", - new BytesArray("{\"script\":\"abc\"}")).build())) + new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON).build())) .build(); DeleteStoredScriptRequest request = new DeleteStoredScriptRequest("_lang", "_id"); @@ -457,7 +458,7 @@ public void testGetStoredScript() throws Exception { .metaData(MetaData.builder() .putCustom(ScriptMetaData.TYPE, new ScriptMetaData.Builder(null).storeScript("_lang", "_id", - new BytesArray("{\"script\":\"abc\"}")).build())) + new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON).build())) .build(); assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id"))); @@ -471,12 +472,14 @@ public void testValidateScriptSize() throws Exception { int maxSize = 0xFFFF; buildScriptService(Settings.EMPTY); // allowed - scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 13) + "\"}")); + scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 13) + "\"}"), + XContentType.JSON); // disallowed IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 12) + "\"}")); + scriptService.validateStoredScript("_id", "test", + new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 12) + "\"}"), XContentType.JSON); }); assertThat(e.getMessage(), equalTo( "Limit of script size in bytes [" + maxSize+ "] has been exceeded for script [_id] with size [" + (maxSize + 1) + "]")); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index d57e2774b55c1..a4a6afa208dff 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.children.Children; @@ -121,8 +122,8 @@ public void setupSuiteScopeCluster() throws Exception { requests.add(client().prepareIndex("test", "article", "b").setSource("category", new String[]{"a", "b"}, "randomized", false)); requests.add(client().prepareIndex("test", "article", "c").setSource("category", new String[]{"a", "b", "c"}, "randomized", false)); requests.add(client().prepareIndex("test", "article", "d").setSource("category", new String[]{"c"}, "randomized", false)); - requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}")); - requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}")); + requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}", XContentType.JSON)); indexRandom(true, requests); ensureSearchable("test"); @@ -240,7 +241,7 @@ public void testWithDeletes() throws Exception { ); List requests = new ArrayList<>(); - requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}")); + requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex(indexName, "child", "0").setParent("1").setSource("count", 1)); requests.add(client().prepareIndex(indexName, "child", "1").setParent("1").setSource("count", 1)); requests.add(client().prepareIndex(indexName, "child", "2").setParent("1").setSource("count", 1)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index 51d702e3548b9..522981b4f01af 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -397,8 +398,8 @@ public void testParentFilterResolvedCorrectly() throws Exception { ensureGreen("idx2"); List indexRequests = new ArrayList<>(2); - indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}")); - indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}")); + indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}", XContentType.JSON)); + indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}", XContentType.JSON)); indexRandom(true, indexRequests); SearchResponse response = client().prepareSearch("idx2").setTypes("provider") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index 764ed3ba73b09..f7e3d9a61b598 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; @@ -99,10 +100,10 @@ private void addTermsDocs(String term, int numInClass, int numNotInClass, List builders) { String sourceClass = "{\"text\": \"" + term + "\"}"; for (int i = 0; i < numDocs; i++) { - builders.add(client().prepareIndex(index, type).setSource(sourceClass)); + builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON)); } } diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index 391d8be412e4f..78cb846932f38 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.HasChildQueryBuilder; @@ -1009,7 +1010,7 @@ public void testParentFieldQuery() throws Exception { .get(); assertHitCount(response, 0L); - client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get(); + client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get(); refresh(); response = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).get(); @@ -1018,7 +1019,7 @@ public void testParentFieldQuery() throws Exception { response = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).get(); assertHitCount(response, 1L); - client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get(); + client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get(); refresh(); response = client().prepareSearch("test").setQuery(termsQuery("_parent#parent", "p1", "p2")).get(); assertHitCount(response, 2L); @@ -1039,13 +1040,13 @@ public void testParentIdQuery() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get(); + client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get(); refresh(); SearchResponse response = client().prepareSearch("test").setQuery(parentId("child", "p1")).get(); assertHitCount(response, 1L); - client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get(); + client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get(); refresh(); response = client().prepareSearch("test") @@ -1405,8 +1406,8 @@ public void testParentChildQueriesViaScrollApi() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); for (int i = 0; i < 10; i++) { - client().prepareIndex("test", "parent", "p" + i).setSource("{}").get(); - client().prepareIndex("test", "child", "c" + i).setSource("{}").setParent("p" + i).get(); + client().prepareIndex("test", "parent", "p" + i).setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "c" + i).setSource("{}", XContentType.JSON).setParent("p" + i).get(); } refresh(); @@ -1488,8 +1489,8 @@ public void testTypeIsAppliedInHasParentInnerQuery() throws Exception { List indexRequests = new ArrayList<>(); indexRequests.add(client().prepareIndex("test", "parent", "1").setSource("field1", "a")); - indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}")); - indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}")); + indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON)); + indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}", XContentType.JSON)); indexRandom(true, indexRequests); SearchResponse searchResponse = client().prepareSearch("test") @@ -1883,8 +1884,8 @@ public void testMinMaxChildren() throws Exception { public void testParentFieldToNonExistingType() { assertAcked(prepareCreate("test").addMapping("parent").addMapping("child", "_parent", "type=parent2")); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); try { @@ -1898,8 +1899,8 @@ public void testParentFieldToNonExistingType() { public void testHasParentInnerQueryType() { assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type")); - client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get(); - client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get(); + client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get(); refresh(); //make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead SearchResponse searchResponse = client().prepareSearch("test").setTypes("child-type").setQuery( @@ -1909,8 +1910,8 @@ public void testHasParentInnerQueryType() { public void testHasChildInnerQueryType() { assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type")); - client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get(); - client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get(); + client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get(); refresh(); //make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead SearchResponse searchResponse = client().prepareSearch("test").setTypes("parent-type").setQuery( diff --git a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java index 5aead911caa57..44b01b3a4b974 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -69,8 +70,8 @@ public void testEagerParentFieldLoading() throws Exception { .addMapping("child", childMapping(false))); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); @@ -84,8 +85,8 @@ public void testEagerParentFieldLoading() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); @@ -101,9 +102,9 @@ public void testEagerParentFieldLoading() throws Exception { // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals // is loaded by the size of the field data cache, because global ordinals on 1 segment shards takes no extra memory. - client().prepareIndex("test", "parent", "1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); refresh(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); @@ -117,8 +118,8 @@ public void testChangingEagerParentFieldLoadingAtRuntime() throws Exception { .addMapping("child", "_parent", "type=parent")); ensureGreen(); - client().prepareIndex("test", "parent", "1").setSource("{}").get(); - client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); + client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get(); + client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get(); refresh(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); @@ -153,7 +154,7 @@ public void run() { // Need to add a new doc otherwise the refresh doesn't trigger a new searcher // Because it ends up in its own segment, but isn't of type parent or child, this doc doesn't contribute to the size of the fielddata cache - client().prepareIndex("test", "dummy", "dummy").setSource("{}").get(); + client().prepareIndex("test", "dummy", "dummy").setSource("{}", XContentType.JSON).get(); refresh(); response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L)); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index c00882522243a..1d24a1f16d5b9 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -331,19 +332,19 @@ public void testRandomParentChild() throws Exception { int[] child2InnerObjects = new int[numDocs]; for (int parent = 0; parent < numDocs; parent++) { String parentId = String.format(Locale.ENGLISH, "%03d", parent); - requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}")); + requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}", XContentType.JSON)); int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs); int limit = child1 + numChildDocs; for (; child1 < limit; child1++) { requestBuilders.add(client().prepareIndex("idx", "child1", - String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}")); + String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}", XContentType.JSON)); } numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs); limit = child2 + numChildDocs; for (; child2 < limit; child2++) { requestBuilders.add(client().prepareIndex("idx", "child2", - String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}")); + String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}", XContentType.JSON)); } } indexRandom(true, requestBuilders); @@ -727,17 +728,26 @@ public void testRoyals() throws Exception { ); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("royals", "king", "king").setSource("{}")); - requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king").setSource("{}")); - requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king").setSource("{}")); + requests.add(client().prepareIndex("royals", "king", "king").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king") + .setSource("{}", XContentType.JSON)); + requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king") + .setSource("{}", XContentType.JSON)); indexRandom(true, requests); SearchResponse response = client().prepareSearch("royals") @@ -899,10 +909,10 @@ public void testMatchesQueriesNestedInnerHits() throws Exception { public void testMatchesQueriesParentChildInnerHits() throws Exception { assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("index", "parent", "1").setSource("{}")); + requests.add(client().prepareIndex("index", "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index", "child", "1").setParent("1").setSource("field", "value1")); requests.add(client().prepareIndex("index", "child", "2").setParent("1").setSource("field", "value2")); - requests.add(client().prepareIndex("index", "parent", "2").setSource("{}")); + requests.add(client().prepareIndex("index", "parent", "2").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index", "child", "3").setParent("2").setSource("field", "value1")); indexRandom(true, requests); @@ -938,7 +948,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { public void testDontExplode() throws Exception { assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); - requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}")); + requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}", XContentType.JSON)); requests.add(client().prepareIndex("index1", "child", "1").setParent("1").setSource("field", "value1")); indexRandom(true, requests); @@ -1025,7 +1035,7 @@ public void testInnerHitsWithIgnoreUnmapped() throws Exception { ); createIndex("index2"); client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get(); - client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}").get(); + client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}", XContentType.JSON).get(); client().prepareIndex("index2", "type", "3").setSource("key", "value").get(); refresh(); diff --git a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 236d5ce1ab8ef..a952ae8639695 100644 --- a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; @@ -689,8 +690,8 @@ public void testGetFieldsComplexField() throws Exception { .endArray() .endObject().bytes(); - client().prepareIndex("my-index", "my-type1", "1").setSource(source).get(); - client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source).get(); + client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get(); + client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source, XContentType.JSON).get(); String field = "field1.field2.field3.field4"; diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index 2707e3c0b275c..568c723f8d16e 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -47,6 +47,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; @@ -230,7 +231,7 @@ public void testShapeRelations() throws Exception { new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close())); BytesReference data = jsonBuilder().startObject().field("area", polygon).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // Point in polygon @@ -292,7 +293,7 @@ public void testShapeRelations() throws Exception { new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close())); data = jsonBuilder().startObject().field("area", inverse).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "2").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "2").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // re-check point on polygon hole @@ -326,7 +327,7 @@ public void testShapeRelations() throws Exception { .coordinate(170, -10).coordinate(190, -10).coordinate(190, 10).coordinate(170, 10).close()); data = jsonBuilder().startObject().field("area", builder).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); // Create a polygon crossing longitude 180 with hole. @@ -335,7 +336,7 @@ public void testShapeRelations() throws Exception { .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(175, -5).coordinate(185, -5).coordinate(185, 5).coordinate(175, 5).close())); data = jsonBuilder().startObject().field("area", builder).endObject().bytes(); - client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet(); + client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet(); client().admin().indices().prepareRefresh().execute().actionGet(); result = client().prepareSearch() @@ -384,7 +385,7 @@ public void testBulk() throws Exception { client().admin().indices().prepareCreate("countries").setSettings(settings) .addMapping("country", xContentBuilder.string()).execute().actionGet(); - BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null).execute().actionGet(); + BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null, xContentBuilder.contentType()).get(); for (BulkItemResponse item : bulk.getItems()) { assertFalse("unable to index data", item.isFailed()); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 067c2dcac0556..7e8aa1ad9d0fe 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.locationtech.spatial4j.shape.Rectangle; import com.vividsolutions.jts.geom.Coordinate; @@ -69,7 +70,8 @@ public void testNullShape() throws Exception { client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); - client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}", XContentType.JSON) + .setRefreshPolicy(IMMEDIATE).get(); GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet(); assertThat(result.getField("location"), nullValue()); } @@ -252,8 +254,8 @@ public void testShapeFetchingPath() throws Exception { .setSource( String.format( Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location - ) - ).setRefreshPolicy(IMMEDIATE).get(); + ), XContentType.JSON) + .setRefreshPolicy(IMMEDIATE).get(); client().prepareIndex("test", "type", "1") .setSource(jsonBuilder().startObject().startObject("location") .field("type", "polygon") diff --git a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java index ccc3102f3a80b..c7e78f2cc28fd 100644 --- a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -59,7 +60,7 @@ public void testSimpleMultiSearchMoreRequests() { createIndex("test"); int numDocs = randomIntBetween(0, 16); for (int i = 0; i < numDocs; i++) { - client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get(); + client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}", XContentType.JSON).get(); } refresh(); diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java index 2fe14e4ee102b..1dbeb24d4ed9e 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; @@ -64,7 +65,7 @@ protected Collection> nodePlugins() { @Before public void setup() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); - prepareCreate("test").setSource(indexBody).get(); + prepareCreate("test").setSource(indexBody, XContentType.JSON).get(); ensureGreen("test"); } @@ -151,7 +152,7 @@ public void testWithLotsOfTypes() throws Exception { public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); - reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody)); + reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON)); indexRandom(true, false, reqs); SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get(); @@ -214,7 +215,7 @@ public void testKeywordWithWhitespace() throws Exception { public void testExplicitAllFieldsRequested() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json"); - prepareCreate("test2").setSource(indexBody).get(); + prepareCreate("test2").setSource(indexBody, XContentType.JSON).get(); ensureGreen("test2"); List reqs = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index ebc2df30e1d3e..69e4899a73d1c 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; @@ -1797,7 +1798,7 @@ public void testRangeQueryWithTimeZone() throws Exception { public void testSearchEmptyDoc() { assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}")); - client().prepareIndex("test", "type1", "1").setSource("{}").get(); + client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON).get(); refresh(); assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 3ed464f312c1e..1bec4bb20bfbe 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; @@ -471,7 +472,7 @@ public void testDocWithAllTypes() throws Exception { List reqs = new ArrayList<>(); String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json"); - reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody)); + reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON)); indexRandom(true, false, reqs); SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get(); diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java index df89bda88d83b..ae8ad66ac8d94 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.search.RestClearScrollAction; @@ -48,7 +49,8 @@ public void testParseClearScrollRequest() throws Exception { public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception { RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java index 7d2a5024cc8de..662bc07f90d7e 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.search.RestSearchScrollAction; @@ -52,7 +53,8 @@ public void testParseSearchScrollRequest() throws Exception { public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception { RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class)); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray("{invalid_json}")).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 099cce038c116..f63f13b6dd24b 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; @@ -289,7 +290,7 @@ public void testSimpleTerminateAfterCount() throws Exception { public void testInsaneFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); @@ -297,7 +298,7 @@ public void testInsaneFromAndSize() throws Exception { public void testTooLargeFromAndSize() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); @@ -307,7 +308,7 @@ public void testTooLargeFromAndSize() throws Exception { public void testLargeFromAndSizeSucceeds() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); @@ -318,7 +319,7 @@ public void testLargeFromAndSizeSucceeds() throws Exception { public void testTooLargeFromAndSizeOkBySetting() throws Exception { prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); @@ -333,7 +334,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); @@ -343,7 +344,7 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); @@ -353,7 +354,7 @@ public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws public void testTooLargeRescoreWindow() throws Exception { createIndex("idx"); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertRescoreWindowFails(Integer.MAX_VALUE); assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); @@ -363,7 +364,7 @@ public void testTooLargeRescoreOkBySetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); prepareCreate("idx").setSettings(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -374,7 +375,7 @@ public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. defaultMaxWindow * 2).get(); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -388,7 +389,7 @@ public void testTooLargeRescoreOkByDynamicSetting() throws Exception { .setSettings( Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), @@ -403,7 +404,7 @@ public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception // Note that this is the RESULT window Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) .get()); - indexRandom(true, client().prepareIndex("idx", "type").setSource("{}")); + indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON)); assertHitCount( client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index 84dd3dabf6b61..7659576555868 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; @@ -91,7 +92,7 @@ public void testIssue8226() { assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test"))); } if (i > 0) { - client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get(); + client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}", XContentType.JSON).get(); } } refresh(); @@ -422,9 +423,9 @@ public void testIssue2986() { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=keyword").get()); - client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet(); - client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet(); - client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet(); + client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}", XContentType.JSON).execute().actionGet(); + client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}", XContentType.JSON).execute().actionGet(); + client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}", XContentType.JSON).execute().actionGet(); refresh(); SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true) .addSort("field1", SortOrder.ASC).execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 5769a8e89f346..a7d0ba481785d 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; @@ -44,7 +43,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index acc896708250d..ff7283f4f665a 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; @@ -533,7 +534,7 @@ public void testIncludeGlobalState() throws Exception { .endObject() .endArray() .endObject().bytes(); - assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource).get()); + assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource, XContentType.JSON).get()); } if(testScript) { diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 2c98b393642ab..bc0658232159e 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.DocumentMissingException; @@ -536,7 +537,7 @@ public void testVersionedUpdate() throws Exception { // With internal versions, tt means "if object is there with version X, update it or explode. If it is not there, index. client().prepareUpdate(indexOrAlias(), "type", "3") .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))) - .setVersion(10).setUpsert("{ \"text\": \"v0\" }").get(); + .setVersion(10).setUpsert("{ \"text\": \"v0\" }", XContentType.JSON).get(); get = get("test", "type", "3"); assertThat(get.getVersion(), equalTo(1L)); assertThat((String) get.getSource().get("text"), equalTo("v0")); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index f14d91465f6e2..beab15165d698 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -245,7 +245,7 @@ private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XCon private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1") - .setDoc(xContentBuilder.bytes().utf8ToString()) + .setDoc(xContentBuilder) .setDocAsUpsert(true) .setFields("_source"); if (detectNoop != null) { diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java index 71ec194866266..c61b736bf6db1 100644 --- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.rest; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; @@ -47,12 +48,14 @@ public void testIndexWithoutId() throws IOException { public void testUpsert() throws IOException { locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{" + "\"doc\": {\"test\": \"test\"}," - + "\"doc_as_upsert\": true}"))); + + "\"doc_as_upsert\": true}", ContentType.APPLICATION_JSON))); } private void locationTestCase(String method, String url) throws IOException { - locationTestCase(client().performRequest(method, url, emptyMap(), new StringEntity("{\"test\": \"test\"}"))); - locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), new StringEntity("{\"test\": \"test\"}"))); + locationTestCase(client().performRequest(method, url, emptyMap(), + new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON))); + locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), + new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON))); } private void locationTestCase(Response response) throws IOException { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java index 71c87937618ba..9ffb14cd9c7d9 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -55,7 +56,7 @@ public void testAllOpsDisabledIndexedScripts() throws IOException { .setId("script1") .setSource(new BytesArray("{\"script\":\"2\"}")) .get(); - client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get(); + client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get(); try { client().prepareUpdate("test", "scriptTest", "1") .setScript(new Script(ScriptType.STORED, ExpressionScriptEngineService.NAME, "script1", Collections.emptyMap())).get(); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java index 3e4471509eea2..7dad299a40f0e 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; @@ -34,7 +35,8 @@ public class MultiSearchTemplateRequestTests extends ESTestCase { public void testParseRequest() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/script/mustache/simple-msearch-template.json"); - RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(data)).build(); + RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withContent(new BytesArray(data), XContentType.JSON).build(); MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 09e160f34f713..97ff20a0360c4 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptType; @@ -178,11 +179,11 @@ public void testIndexedTemplateClient() throws Exception { assertNotNull(getResponse.getStoredScript()); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); @@ -247,11 +248,11 @@ public void testIndexedTemplate() throws Exception { ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); @@ -348,11 +349,11 @@ public void testIndexedTemplateWithArray() throws Exception { .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes()) ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON)); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON)); bulkRequestBuilder.get(); client().admin().indices().prepareRefresh().get(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 1cce5fdbbdbf2..962c453727314 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; @@ -263,7 +264,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { .must(matchQuery("field2", "value")) ).endObject()).get(); - client().prepareIndex("test", "type", "1").setSource("{}").get(); + client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get(); client().prepareIndex("test", "type", "2").setSource("field1", "value").get(); client().prepareIndex("test", "type", "3").setSource("field1", "value", "field2", "value").get(); client().admin().indices().prepareRefresh().get(); @@ -305,7 +306,7 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) .get(); - client().prepareIndex("test", "type", "1").setSource("{}").get(); + client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get(); client().admin().indices().prepareRefresh().get(); logger.info("percolating empty doc with source disabled"); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java index 2905f5b39cf98..75fda55c54987 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java @@ -170,6 +170,22 @@ static Map scrollParams(TimeValue keepAlive) { } static HttpEntity scrollEntity(String scroll) { - return new StringEntity(scroll, ContentType.TEXT_PLAIN); + try (XContentBuilder entity = JsonXContent.contentBuilder()) { + return new StringEntity(entity.startObject() + .field("scroll_id", scroll) + .endObject().string(), ContentType.APPLICATION_JSON); + } catch (IOException e) { + throw new ElasticsearchException("failed to build scroll entity", e); + } + } + + static HttpEntity clearScrollEntity(String scroll) { + try (XContentBuilder entity = JsonXContent.contentBuilder()) { + return new StringEntity(entity.startObject() + .array("scroll_id", scroll) + .endObject().string(), ContentType.APPLICATION_JSON); + } catch (IOException e) { + throw new ElasticsearchException("failed to build clear scroll entity", e); + } } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java index 28425d9145c72..4aa64f5660ef1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -57,6 +57,7 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; +import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.clearScrollEntity; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams; import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath; @@ -110,7 +111,7 @@ protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Cons @Override protected void clearScroll(String scrollId, Runnable onCompletion) { - client.performRequestAsync("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() { + client.performRequestAsync("DELETE", scrollPath(), emptyMap(), clearScrollEntity(scrollId), new ResponseListener() { @Override public void onSuccess(org.elasticsearch.client.Response response) { logger.debug("Successfully cleared [{}]", scrollId); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index bbda7efd67b0b..65aab6cf07b6e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; @@ -201,7 +202,7 @@ public void testUpdateByQueryCancel() throws Exception { " \"test\" : {}\n" + " } ]\n" + "}"); - assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get()); + assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX), (response, total, modified) -> { assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request"))); @@ -234,7 +235,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { " \"test\" : {}\n" + " } ]\n" + "}"); - assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get()); + assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5), (response, total, modified) -> { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java index abef7fb590294..b190aee3619a8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java @@ -143,7 +143,7 @@ public void testPipelineQueryParameterIsError() throws IOException { body.endObject(); } body.endObject(); - request.withContent(body.bytes()); + request.withContent(body.bytes(), body.contentType()); } request.withParams(singletonMap("pipeline", "doesn't matter")); Exception e = expectThrows(IllegalArgumentException.class, () -> action.buildRequest(request.build())); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java index 3de7e09debe24..a5b19fea94e79 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java @@ -186,7 +186,8 @@ public void testScrollParams() { public void testScrollEntity() throws IOException { String scroll = randomAsciiOfLength(30); HttpEntity entity = scrollEntity(scroll); - assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue()); - assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))); + assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue()); + assertThat(Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)), + containsString("\"" + scroll + "\"")); } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index 10a2da21ac948..c3b37ef7be84d 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -21,6 +21,7 @@ import io.netty.channel.Channel; import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; import org.elasticsearch.common.bytes.BytesArray; @@ -30,7 +31,12 @@ import org.elasticsearch.transport.netty4.Netty4Utils; import java.net.SocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class Netty4HttpRequest extends RestRequest { @@ -39,7 +45,7 @@ public class Netty4HttpRequest extends RestRequest { private final BytesReference content; Netty4HttpRequest(NamedXContentRegistry xContentRegistry, FullHttpRequest request, Channel channel) { - super(xContentRegistry, request.uri()); + super(xContentRegistry, request.uri(), new HttpHeadersMap(request.headers())); this.request = request; this.channel = channel; if (request.content().isReadable()) { @@ -120,14 +126,96 @@ public Channel getChannel() { return channel; } - @Override - public String header(String name) { - return request.headers().get(name); - } + /** + * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications + * and due to the underlying implementation, it performs case insensitive lookups of key to values. + * + * It is important to note that this implementation does have some downsides in that each invocation of the + * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a + * view of the underlying values. + */ + private static class HttpHeadersMap implements Map> { - @Override - public Iterable> headers() { - return request.headers().entries(); - } + private final HttpHeaders httpHeaders; + + private HttpHeadersMap(HttpHeaders httpHeaders) { + this.httpHeaders = httpHeaders; + } + + @Override + public int size() { + return httpHeaders.size(); + } + + @Override + public boolean isEmpty() { + return httpHeaders.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return key instanceof String && httpHeaders.contains((String) key); + } + + @Override + public boolean containsValue(Object value) { + return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals); + } + @Override + public List get(Object key) { + return key instanceof String ? httpHeaders.getAll((String) key) : null; + } + + @Override + public List put(String key, List value) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public List remove(Object key) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public void putAll(Map> m) { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public void clear() { + throw new UnsupportedOperationException("modifications are not supported"); + } + + @Override + public Set keySet() { + return httpHeaders.names(); + } + + @Override + public Collection> values() { + return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList()); + } + + @Override + public Set>> entrySet() { + return httpHeaders.names().stream().map(k -> new Entry>() { + + @Override + public String getKey() { + return k; + } + + @Override + public List getValue() { + return httpHeaders.getAll(k); + } + + @Override + public List setValue(List value) { + throw new UnsupportedOperationException("modifications are not supported"); + } + }).collect(Collectors.toSet()); + } + } } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java index 959a4d7c9f030..1050404563486 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java @@ -122,6 +122,7 @@ private Collection processRequestsWithBody(HttpMethod method, HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content); request.headers().add(HttpHeaderNames.HOST, "localhost"); request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()); + request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json"); requests.add(request); } return sendRequests(remoteAddress, requests); diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index c4de761c8e798..92cbb2e6e397b 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugin.mapper.MapperSizePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -107,7 +108,7 @@ public void testBasic() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true")); final String source = "{\"f\":10}"; indexRandom(true, - client().prepareIndex("test", "type", "1").setSource(source)); + client().prepareIndex("test", "type", "1").setSource(source, XContentType.JSON)); GetResponse getResponse = client().prepareGet("test", "type", "1").setStoredFields("_size").get(); assertNotNull(getResponse.getField("_size")); assertEquals(source.length(), getResponse.getField("_size").getValue()); diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle index fceeac283fc3a..82fd14c1e15f4 100644 --- a/qa/backwards-5.0/build.gradle +++ b/qa/backwards-5.0/build.gradle @@ -38,7 +38,7 @@ integTest { cluster { numNodes = 4 numBwcNodes = 2 - bwcVersion = "5.3.0-SNAPSHOT" + bwcVersion = "6.0.0-alpha1-SNAPSHOT" // TODO switch back to 5.3 and remove blacklist once backported setting 'logger.org.elasticsearch', 'DEBUG' } } diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 1e3ae7aaa9996..92328df766b10 100644 --- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.backwards; import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; @@ -70,7 +71,7 @@ private void ensureGreen() throws IOException { private void createIndex(String name, Settings settings) throws IOException { assertOK(client().performRequest("PUT", name, Collections.emptyMap(), - new StringEntity("{ \"settings\": " + Strings.toString(settings) + " }"))); + new StringEntity("{ \"settings\": " + Strings.toString(settings) + " }", ContentType.APPLICATION_JSON))); } private void updateIndexSetting(String name, Settings.Builder settings) throws IOException { @@ -85,13 +86,14 @@ protected int indexDocs(String index, final int idStart, final int numDocs) thro for (int i = 0; i < numDocs; i++) { final int id = idStart + i; assertOK(client().performRequest("PUT", index + "/test/" + id, emptyMap(), - new StringEntity("{\"test\": \"test_" + id + "\"}"))); + new StringEntity("{\"test\": \"test_" + id + "\"}", ContentType.APPLICATION_JSON))); } return numDocs; } public void testSeqNoCheckpoints() throws Exception { Nodes nodes = buildNodeAndVersions(); + assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered: {}", nodes.toString()); final String bwcNames = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.joining(",")); Settings.Builder settings = Settings.builder() diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java index 2cf681e9e5a85..51dbad22dec76 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -54,7 +53,7 @@ public void testThatErrorTraceParamReturns400() throws IOException { fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); - assertThat(response.getHeader("Content-Type"), is("application/json")); + assertThat(response.getHeader("Content-Type"), is("application/json; charset=UTF-8")); assertThat(EntityUtils.toString(e.getResponse().getEntity()), is("{\"error\":\"error traces in responses are disabled.\"}")); assertThat(response.getStatusLine().getStatusCode(), is(400)); } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java index cd3cf1a38d8b2..d0b80595a26ee 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java @@ -22,7 +22,6 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; import java.util.Collections; @@ -52,7 +51,7 @@ public void testThatErrorTraceWorksByDefault() throws IOException { fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); - assertThat(response.getHeader("Content-Type"), containsString("application/json")); + assertThat(response.getHeader("Content-Type"), containsString("application/json; charset=UTF-8")); assertThat(EntityUtils.toString(response.getEntity()), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + "nested: ActionRequestValidationException[Validation Failed: 1:"))); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 038477cc3aa82..c906330fe598d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -96,6 +96,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.discovery.Discovery; @@ -1226,10 +1227,10 @@ protected final IndexResponse index(String index, String type, String id, Object * return client().prepareIndex(index, type, id).setSource(source).execute().actionGet(); * *

- * where source is a String. + * where source is a JSON String. */ protected final IndexResponse index(String index, String type, String id, String source) { - return client().prepareIndex(index, type, id).setSource(source).execute().actionGet(); + return client().prepareIndex(index, type, id).setSource(source, XContentType.JSON).execute().actionGet(); } /** @@ -1387,7 +1388,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet()); String index = RandomPicks.randomFrom(random, indices); bogusIds.add(new Tuple<>(index, id)); - builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}")); + builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON)); } } final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java index 3d1ce29143208..ee00fedca31cd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java @@ -19,9 +19,9 @@ package org.elasticsearch.test.rest; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -53,8 +53,8 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - return super.newBuilder(autoDetectSource, useFiltering); + public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { + return super.newBuilder(xContentType, useFiltering); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java index ae8c4c82c4663..83caf0293e0ab 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java @@ -21,28 +21,31 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestRequest; +import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class FakeRestRequest extends RestRequest { - private final Map headers; private final BytesReference content; private final Method method; - + private final SocketAddress remoteAddress; public FakeRestRequest() { - this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/"); + this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/", null); } - private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map headers, Map params, - BytesReference content, Method method, String path) { - super(xContentRegistry, params, path); - this.headers = headers; + private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map> headers, Map params, + BytesReference content, Method method, String path, SocketAddress remoteAddress) { + super(xContentRegistry, params, path, headers); this.content = content; this.method = method; + this.remoteAddress = remoteAddress; } @Override @@ -66,19 +69,14 @@ public BytesReference content() { } @Override - public String header(String name) { - return headers.get(name); - } - - @Override - public Iterable> headers() { - return headers.entrySet(); + public SocketAddress getRemoteAddress() { + return remoteAddress; } public static class Builder { private final NamedXContentRegistry xContentRegistry; - private Map headers = new HashMap<>(); + private Map> headers = new HashMap<>(); private Map params = new HashMap<>(); @@ -88,11 +86,13 @@ public static class Builder { private Method method = Method.GET; + private SocketAddress address = null; + public Builder(NamedXContentRegistry xContentRegistry) { this.xContentRegistry = xContentRegistry; } - public Builder withHeaders(Map headers) { + public Builder withHeaders(Map> headers) { this.headers = headers; return this; } @@ -102,8 +102,11 @@ public Builder withParams(Map params) { return this; } - public Builder withContent(BytesReference content) { + public Builder withContent(BytesReference content, XContentType xContentType) { this.content = content; + if (xContentType != null) { + headers.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + } return this; } @@ -117,8 +120,13 @@ public Builder withMethod(Method method) { return this; } + public Builder withRemoteAddress(SocketAddress address) { + this.address = address; + return this; + } + public FakeRestRequest build() { - return new FakeRestRequest(xContentRegistry, headers, params, content, method, path); + return new FakeRestRequest(xContentRegistry, headers, params, content, method, path, address); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index c18dad907bd6e..565aad7ffbbac 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -122,6 +122,7 @@ public ClientYamlTestResponse callApi(String apiName, Map params if (supportedMethods.contains("GET") && RandomizedTest.rarely()) { logger.debug("sending the request body as source param with GET method"); queryStringParams.put("source", body); + queryStringParams.put("source_type", ContentType.APPLICATION_JSON.toString()); requestMethod = "GET"; } else { requestMethod = RandomizedTest.randomFrom(supportedMethods); From c53e296bb2f020e150add94bb244ef11531dd4e8 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 20 Jan 2017 07:49:22 -0500 Subject: [PATCH 02/34] start addressing review feedback --- .../storedscripts/PutStoredScriptRequest.java | 18 ++++++--- .../PutStoredScriptRequestBuilder.java | 7 ++++ .../PutMappingClusterStateUpdateRequest.java | 15 +++++-- .../mapping/put/PutMappingRequest.java | 39 ++++++++++++------- .../put/TransportPutMappingAction.java | 2 +- .../template/put/PutIndexTemplateRequest.java | 2 +- .../metadata/MetaDataMappingService.java | 4 +- .../common/xcontent/XContentType.java | 22 +++++++++++ .../index/mapper/DocumentMapper.java | 4 ++ .../index/mapper/DocumentMapperParser.java | 13 +++++-- .../index/mapper/MapperService.java | 13 ++++--- .../mapping/put/PutMappingRequestTests.java | 26 +++++-------- .../metadata/MetaDataMappingServiceTests.java | 9 ++++- .../mapper/MultiFieldCopyToMapperTests.java | 2 +- .../MultiFieldIncludeInAllMapperTests.java | 2 +- .../index/mapper/SourceFieldMapperTests.java | 9 +++-- .../elasticsearch/script/StoredScriptsIT.java | 7 ++-- .../metrics/ScriptedMetricIT.java | 9 +++-- .../aggregations/pipeline/BucketScriptIT.java | 3 +- .../pipeline/BucketSelectorIT.java | 4 +- .../SharedClusterSnapshotRestoreIT.java | 2 +- .../expression/IndexedExpressionTests.java | 2 +- .../script/mustache/SearchTemplateIT.java | 17 ++++---- 23 files changed, 154 insertions(+), 77 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 079dfb810242b..9b7d0feb9e508 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -25,10 +25,12 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -99,15 +101,23 @@ public XContentType xContentType() { return xContentType; } + /** + * Set the script source using bytes. + * @deprecated this method is deprecated as it relies on content type detection. Use {@link #script(BytesReference, XContentType)} + */ @Deprecated public PutStoredScriptRequest script(BytesReference source) { this.script = source; + this.xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); return this; } + /** + * Set the script source and the content type of the bytes. + */ public PutStoredScriptRequest script(BytesReference source, XContentType xContentType) { this.script = source; - this.xContentType = xContentType; + this.xContentType = Objects.requireNonNull(xContentType); return this; } @@ -142,11 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { public String toString() { String sSource = "_na_"; try { - if (xContentType == null) { - sSource = XContentHelper.convertToJson(script, false); - } else { - sSource = XContentHelper.convertToJson(script, false, xContentType); - } + sSource = XContentHelper.convertToJson(script, false, xContentType); } catch (Exception e) { // ignore } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java index 366b3923223cc..5f235b8966a9b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java @@ -41,12 +41,19 @@ public PutStoredScriptRequestBuilder setId(String id) { return this; } + /** + * Set the source of the script. + * @deprecated this method requires content type detection. Use {@link #setSource(BytesReference, XContentType)} instead + */ @Deprecated public PutStoredScriptRequestBuilder setSource(BytesReference source) { request.script(source); return this; } + /** + * Set the source of the script along with the content type of the source + */ public PutStoredScriptRequestBuilder setSource(BytesReference source, XContentType xContentType) { request.script(source, xContentType); return this; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java index 0f396afa5513b..a05ee4773aee9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java @@ -20,6 +20,8 @@ package org.elasticsearch.action.admin.indices.mapping.put; import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; /** * Cluster state update request that allows to put a mapping @@ -28,10 +30,12 @@ public class PutMappingClusterStateUpdateRequest extends IndicesClusterStateUpda private String type; - private String source; + private BytesReference source; private boolean updateAllTypes = false; + private XContentType xContentType; + public PutMappingClusterStateUpdateRequest() { } @@ -45,12 +49,17 @@ public PutMappingClusterStateUpdateRequest type(String type) { return this; } - public String source() { + public BytesReference source() { return source; } - public PutMappingClusterStateUpdateRequest source(String source) { + public XContentType xContentType() { + return xContentType; + } + + public PutMappingClusterStateUpdateRequest source(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; return this; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 025a47c47fcf6..79a42a1dc8eb8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -29,16 +29,15 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; @@ -70,7 +69,8 @@ public class PutMappingRequest extends AcknowledgedRequest im private String type; - private String source; + private BytesReference source; + private XContentType xContentType; private boolean updateAllTypes = false; private Index concreteIndex; @@ -96,7 +96,7 @@ public ActionRequestValidationException validate() { } if (source == null) { validationException = addValidationError("mapping source is missing", validationException); - } else if (source.isEmpty()) { + } else if (source == null || source.length() == 0) { validationException = addValidationError("mapping source is empty", validationException); } if (concreteIndex != null && (indices != null && indices.length > 0)) { @@ -167,10 +167,17 @@ public PutMappingRequest type(String type) { /** * The mapping source definition. */ - public String source() { + public BytesReference source() { return source; } + /** + * The type of content held in the source + */ + public XContentType getXContentType() { + return xContentType; + } + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). @@ -283,11 +290,8 @@ public PutMappingRequest source(String mappingSource) { * The mapping source definition. */ public PutMappingRequest source(String mappingSource, XContentType xContentType) { - try { - this.source = XContentHelper.convertToJson(new BytesArray(mappingSource.getBytes(StandardCharsets.UTF_8)), false, xContentType); - } catch (IOException e) { - throw new UncheckedIOException("failed to convert mapping source to json", e); - } + this.source = new BytesArray(mappingSource.getBytes(StandardCharsets.UTF_8)); + this.xContentType = Objects.requireNonNull(xContentType); return this; } @@ -309,9 +313,11 @@ public void readFrom(StreamInput in) throws IOException { indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - source = in.readString(); + source = in.readBytesReference(); + xContentType = XContentType.readFrom(in); } else { - source = XContentHelper.convertToJson(new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)), false); + source = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); + xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); } updateAllTypes = in.readBoolean(); readTimeout(in); @@ -324,7 +330,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArrayNullable(indices); indicesOptions.writeIndicesOptions(out); out.writeOptionalString(type); - out.writeString(source); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + out.writeBytesReference(source); + xContentType.writeTo(out); + } else if (xContentType.hasStringRepresentation()) { + out.writeString(source.utf8ToString()); + } else { + throw new IllegalStateException("cannot send [" + xContentType + "] to an older node"); + } out.writeBoolean(updateAllTypes); writeTimeout(out); out.writeOptionalWriteable(concreteIndex); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index d9ebf88fda6d7..9433259580b4d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -83,7 +83,7 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) .indices(concreteIndices).type(request.type()) .updateAllTypes(request.updateAllTypes()) - .source(request.source()); + .source(request.source(), request.getXContentType()); metaDataMappingService.putMapping(updateRequest, new ActionListener() { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 4af876f6293b7..821023bfd1ed8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -197,7 +197,7 @@ public PutIndexTemplateRequest settings(String source, XContentType xContentType } /** - * The settings to crete the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml/properties format). */ public PutIndexTemplateRequest settings(Map source) { try { diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index c0032a4b6a452..8e7ae4f33a54f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -259,9 +259,9 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt DocumentMapper existingMapper = mapperService.documentMapper(request.type()); if (MapperService.DEFAULT_MAPPING.equals(request.type())) { // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default - newMapper = mapperService.parse(request.type(), mappingUpdateSource, false); + newMapper = mapperService.parse(request.type(), mappingUpdateSource, false, request.xContentType()); } else { - newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null); + newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null, request.xContentType()); if (existingMapper != null) { // first, simulate: just call merge and ignore the result existingMapper.merge(newMapper.mapping(), request.updateAllTypes()); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index e79713ee52097..09cf5a6f2ac6a 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -58,6 +58,11 @@ public String shortName() { public XContent xContent() { return JsonXContent.jsonXContent; } + + @Override + public boolean hasStringRepresentation() { + return true; + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -77,6 +82,11 @@ public String shortName() { public XContent xContent() { return SmileXContent.smileXContent; } + + @Override + public boolean hasStringRepresentation() { + return false; + } }, /** * A YAML based content type. @@ -96,6 +106,11 @@ public String shortName() { public XContent xContent() { return YamlXContent.yamlXContent; } + + @Override + public boolean hasStringRepresentation() { + return true; + } }, /** * A CBOR based content type. @@ -115,6 +130,11 @@ public String shortName() { public XContent xContent() { return CborXContent.cborXContent; } + + @Override + public boolean hasStringRepresentation() { + return false; + } }; public static XContentType fromMediaTypeOrFormat(String mediaType) { @@ -169,6 +189,8 @@ public String mediaType() { public abstract String mediaTypeWithoutParameters(); + public abstract boolean hasStringRepresentation(); + public static XContentType readFrom(StreamInput in) throws IOException { int index = in.readVInt(); for (XContentType contentType : values()) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 7e0f633e45dd2..a75466f42b4cb 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -196,6 +196,10 @@ public CompressedXContent mappingSource() { return this.mappingSource; } + public XContentType mappingSourceXContentType() { + return XContentType.JSON; + } + public RootObjectMapper root() { return mapping.root; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 031f139075643..8146e79574eae 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.QueryShardContext; @@ -74,13 +75,19 @@ public Mapper.TypeParser.ParserContext parserContext(String type) { } public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { - return parse(type, source, null); + return parse(type, source, null, null); } - public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { + public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource, + XContentType xContentType) throws MapperParsingException { Map mapping = null; if (source != null) { - Map root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); + Map root; + if (xContentType != null) { + root = XContentHelper.convertToMap(source.compressedReference(), true, xContentType).v2(); + } else { + root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); + } Tuple> t = extractMapping(type, root); type = t.v1(); mapping = t.v2(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index ee4b5fea15e0d..40b2d980226f4 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -318,7 +319,8 @@ private synchronized Map internalMerge(Map Source [" @@ -614,8 +616,9 @@ private void checkPartitionedIndexConstraints(DocumentMapper newMapper) { } } - public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException { - return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null); + public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault, + XContentType xContentType) throws MapperParsingException { + return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null, xContentType); } public boolean hasMapping(String mappingType) { @@ -654,7 +657,7 @@ public DocumentMapperForType documentMapperWithAutoCreate(String type) { throw new TypeMissingException(index(), new IllegalStateException("trying to auto create mapping, but dynamic mapping is disabled"), type); } - mapper = parse(type, null, true); + mapper = parse(type, null, true, null); return new DocumentMapperForType(mapper, mapper.mapping()); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 6fea38d0e2821..a9fd6c5450039 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; @@ -76,11 +77,12 @@ public void testPutMappingRequestFromOldVersion() throws IOException { // this is hacky but here goes PutMappingRequest request = new PutMappingRequest("foo"); String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); - request.source(mapping, XContentType.JSON); // THIS IS NOT A BUG! Intentionally specifying the wrong type so we serialize it - assertEquals(mapping, request.source()); + request.source(mapping, XContentType.JSON); + assertEquals(mapping, request.source().utf8ToString()); // output version doesn't matter BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + bytesStreamOutput.setVersion(Version.V_5_0_0); request.writeTo(bytesStreamOutput); StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); @@ -88,25 +90,17 @@ public void testPutMappingRequestFromOldVersion() throws IOException { PutMappingRequest serialized = new PutMappingRequest(); serialized.readFrom(in); - // yaml is translated to JSON on reading - String source = serialized.source(); - assertNotEquals(mapping, source); - assertTrue(source.startsWith("{")); + BytesReference source = serialized.source(); + assertEquals(mapping, source.utf8ToString()); // reading from a fixed version does no translation + bytesStreamOutput = new BytesStreamOutput(); + request.writeTo(bytesStreamOutput); in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); assertEquals(Version.CURRENT, in.getVersion()); serialized = new PutMappingRequest(); serialized.readFrom(in); - assertEquals(mapping, serialized.source()); - } - - public void testPutMappingRequestTranslatesNonJsonToJson() throws IOException { - // this is hacky but here goes - PutMappingRequest request = new PutMappingRequest("foo"); - String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); - request.source(mapping, XContentType.YAML); - assertNotEquals(mapping, request.source()); - assertTrue(request.source().startsWith("{")); + assertEquals(mapping, serialized.source().utf8ToString()); + assertEquals(XContentType.JSON, serialized.getXContentType()); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java index c3e544f9b2b84..a6a270c57ff57 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java @@ -21,13 +21,16 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.hamcrest.Matchers.equalTo; @@ -87,7 +90,8 @@ public void testMappingClusterStateUpdateDoesntChangeExistingIndices() throws Ex final ClusterService clusterService = getInstanceFromNode(ClusterService.class); // TODO - it will be nice to get a random mapping generator final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type"); - request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}"); + request.source(new BytesArray("{ \"properties\" { \"field\": { \"type\": \"string\" }}}".getBytes(StandardCharsets.UTF_8)), + XContentType.JSON); mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request)); assertThat(indexService.mapperService().documentMapper("type").mappingSource(), equalTo(currentMapping)); } @@ -98,7 +102,8 @@ public void testClusterStateIsNotChangedWithIdenticalMappings() throws Exception final MetaDataMappingService mappingService = getInstanceFromNode(MetaDataMappingService.class); final ClusterService clusterService = getInstanceFromNode(ClusterService.class); final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type"); - request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}"); + request.source(new BytesArray("{ \"properties\" { \"field\": { \"type\": \"string\" }}}".getBytes(StandardCharsets.UTF_8)), + XContentType.JSON); ClusterState result = mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request)) .resultingState; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java index 44484530856bc..61e936b269e88 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java @@ -39,7 +39,7 @@ public void testExceptionForCopyToInMultiFields() throws IOException { // first check that for newer versions we throw exception if copy_to is found withing multi field MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY); try { - mapperService.parse("type", new CompressedXContent(mapping.string()), true); + mapperService.parse("type", new CompressedXContent(mapping.string()), true, mapping.contentType()); fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field"); } catch (MapperParsingException e) { assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field.")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java index 8c6ee8da04220..644df1839f4d5 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java @@ -36,7 +36,7 @@ public void testExceptionForIncludeInAllInMultiFields() throws IOException { // first check that for newer versions we throw exception if include_in_all is found withing multi field MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY); Exception e = expectThrows(MapperParsingException.class, () -> - mapperService.parse("type", new CompressedXContent(mapping.string()), true)); + mapperService.parse("type", new CompressedXContent(mapping.string()), true, mapping.contentType())); assertEquals("include_in_all in multi fields is not allowed. Found the include_in_all in field [c] which is within a multi field.", e.getMessage()); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index adf05b9b33817..aff1a9583149e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -113,11 +113,11 @@ public void testDefaultMappingAndNoMapping() throws Exception { .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper mapper = parser.parse("my_type", null, defaultMapping); + DocumentMapper mapper = parser.parse("my_type", null, defaultMapping, null); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); try { - mapper = parser.parse(null, null, defaultMapping); + mapper = parser.parse(null, null, defaultMapping, null); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); fail(); @@ -125,7 +125,7 @@ public void testDefaultMappingAndNoMapping() throws Exception { // all is well } try { - mapper = parser.parse(null, new CompressedXContent("{}"), defaultMapping); + mapper = parser.parse(null, new CompressedXContent("{}"), defaultMapping, XContentType.JSON); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); fail(); @@ -144,7 +144,8 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception { .startObject("_source").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", new CompressedXContent(mapping), defaultMapping); + DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser() + .parse("my_type", new CompressedXContent(mapping), defaultMapping, XContentType.JSON); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(true)); } diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java index 2bc58bb0d22dd..9a0ac6c4db555 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -52,7 +53,7 @@ public void testBasics() { assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(LANG) .setId("foobar") - .setSource(new BytesArray("{\"script\":\"1\"}"))); + .setSource(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON)); String script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar") .get().getStoredScript(); assertNotNull(script); @@ -68,7 +69,7 @@ public void testBasics() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript() .setScriptLang("lang#") .setId("id#") - .setSource(new BytesArray("{}")) + .setSource(new BytesArray("{}"), XContentType.JSON) .get()); assertEquals("Validation Failed: 1: id can't contain: '#';2: lang can't contain: '#';", e.getMessage()); } @@ -77,7 +78,7 @@ public void testMaxScriptSize() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript() .setScriptLang(LANG) .setId("foobar") - .setSource(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1))) + .setSource(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)), XContentType.JSON) .get() ); assertEquals("Limit of script size in bytes [64] has been exceeded for script [foobar] with size [65]", e.getMessage()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index 545c10bcb031a..1bd360c68d98b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; @@ -230,22 +231,22 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(CustomScriptPlugin.NAME) .setId("initScript_stored") - .setSource(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"))); + .setSource(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(CustomScriptPlugin.NAME) .setId("mapScript_stored") - .setSource(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"))); + .setSource(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(CustomScriptPlugin.NAME) .setId("combineScript_stored") - .setSource(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"))); + .setSource(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(CustomScriptPlugin.NAME) .setId("reduceScript_stored") - .setSource(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"))); + .setSource(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"), XContentType.JSON)); indexRandom(true, builders); ensureSearchable(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java index e76b02a8c9521..00066fdacf15f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; @@ -482,7 +483,7 @@ public void testStoredScript() { .setId("my_script") .setScriptLang(CustomScriptPlugin.NAME) // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin - .setSource(new BytesArray("{ \"script\": \"my_script\" }"))); + .setSource(new BytesArray("{ \"script\": \"my_script\" }"), XContentType.JSON)); SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java index 67261ee02a495..9b489bbdf88ad 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; @@ -434,7 +435,8 @@ public void testStoredScript() { .setId("my_script") .setScriptLang(CustomScriptPlugin.NAME) // Source is not interpreted but my_script is defined in CustomScriptPlugin - .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"))); + .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"), + XContentType.JSON)); Script script = new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "my_script", Collections.emptyMap()); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index ff7283f4f665a..46f94f89ed8a1 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -542,7 +542,7 @@ public void testIncludeGlobalState() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MockScriptEngine.NAME) .setId("foobar") - .setSource(new BytesArray("{\"script\":\"1\"}"))); + .setSource(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON)); } logger.info("--> snapshot without global state"); diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java index 9ffb14cd9c7d9..9a35812a9f49c 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java @@ -54,7 +54,7 @@ public void testAllOpsDisabledIndexedScripts() throws IOException { client().admin().cluster().preparePutStoredScript() .setScriptLang(ExpressionScriptEngineService.NAME) .setId("script1") - .setSource(new BytesArray("{\"script\":\"2\"}")) + .setSource(new BytesArray("{\"script\":\"2\"}"), XContentType.JSON) .get(); client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get(); try { diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 97ff20a0360c4..1bab1d5cf6027 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -160,7 +160,7 @@ public void testIndexedTemplateClient() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}"))); + "}"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() @@ -172,7 +172,7 @@ public void testIndexedTemplateClient() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}"))); + "}"), XContentType.JSON)); GetStoredScriptResponse getResponse = client().admin().cluster() .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get(); @@ -222,7 +222,7 @@ public void testIndexedTemplate() throws Exception { " }" + "}" + "}" - )) + ), XContentType.JSON) ); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) @@ -234,7 +234,7 @@ public void testIndexedTemplate() throws Exception { " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + - "}")) + "}"), XContentType.JSON) ); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) @@ -244,7 +244,7 @@ public void testIndexedTemplate() throws Exception { " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + - "}")) + "}"), XContentType.JSON) ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); @@ -313,7 +313,7 @@ public void testIndexedTemplateOverwrite() throws Exception { .setId("git01") .setSource(new BytesArray("{\"template\":{\"query\": {\"match_phrase_prefix\": " + "{\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + - "\"unsupported\": \"unsupported\"}}}}}"))); + "\"unsupported\": \"unsupported\"}}}}}"), XContentType.JSON)); GetStoredScriptResponse getResponse = client().admin().cluster() .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get(); @@ -331,7 +331,8 @@ public void testIndexedTemplateOverwrite() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) .setId("git01") - .setSource(new BytesArray("{\"query\": {\"match_phrase_prefix\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"}}}}"))); + .setSource(new BytesArray("{\"query\": {\"match_phrase_prefix\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"}}}}"), + XContentType.JSON)); SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()) .setRequest(new SearchRequest("testindex").types("test")) @@ -346,7 +347,7 @@ public void testIndexedTemplateWithArray() throws Exception { client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) .setId("4") - .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes()) + .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON) ); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); From 944872906c44b8a56cf72177ce12a5fa319659f8 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 20 Jan 2017 09:37:31 -0500 Subject: [PATCH 03/34] review updates for index templates --- .../CreateIndexClusterStateUpdateRequest.java | 9 ++- .../indices/create/CreateIndexRequest.java | 80 ++++++++++++++----- .../create/CreateIndexRequestBuilder.java | 12 ++- .../template/put/PutIndexTemplateRequest.java | 59 +++++++------- .../put/PutIndexTemplateRequestBuilder.java | 10 ++- .../metadata/IndexTemplateMetaData.java | 7 +- .../metadata/MetaDataCreateIndexService.java | 17 +++- .../MetaDataIndexTemplateService.java | 18 +++-- .../common/compress/CompressedXContent.java | 9 +++ .../index/mapper/MapperService.java | 6 +- .../CreateIndexRequestBuilderTests.java | 2 +- .../indices/template/BWCTemplateTests.java | 12 +-- .../MetaDataIndexTemplateServiceTests.java | 12 +-- .../put/PutIndexTemplateRequestTests.java | 21 ++--- .../index/mapper/MapperServiceTests.java | 6 +- .../template/SimpleIndexTemplateIT.java | 10 ++- .../SignificantTermsSignificanceScoreIT.java | 3 +- .../search/preference/SearchPreferenceIT.java | 5 +- .../search/query/SearchQueryIT.java | 2 +- .../SharedSignificantTermsTestMethods.java | 3 +- 20 files changed, 194 insertions(+), 109 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index a2290a5e2556e..9a73e99acf0f9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -24,7 +24,10 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.transport.TransportMessage; @@ -49,7 +52,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private Settings settings = Settings.Builder.EMPTY_SETTINGS; - private final Map mappings = new HashMap<>(); + private final Map> mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -74,7 +77,7 @@ public CreateIndexClusterStateUpdateRequest settings(Settings settings) { return this; } - public CreateIndexClusterStateUpdateRequest mappings(Map mappings) { + public CreateIndexClusterStateUpdateRequest mappings(Map> mappings) { this.mappings.putAll(mappings); return this; } @@ -129,7 +132,7 @@ public Settings settings() { return settings; } - public Map mappings() { + public Map> mappings() { return mappings; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index a6cc3e9ced8b8..49fd0044c1994 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -32,6 +33,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -47,6 +49,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -71,7 +74,7 @@ public class CreateIndexRequest extends AcknowledgedRequest private Settings settings = EMPTY_SETTINGS; - private final Map mappings = new HashMap<>(); + private final Map> mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -169,19 +172,29 @@ public CreateIndexRequest settings(Settings.Builder settings) { } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #source(String, XContentType)} instead to avoid content type detection */ + @Deprecated public CreateIndexRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Allows to set the settings using a json builder. */ public CreateIndexRequest settings(XContentBuilder builder) { try { - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate json settings from builder", e); } @@ -196,7 +209,7 @@ public CreateIndexRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -208,12 +221,31 @@ public CreateIndexRequest settings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, BytesReference, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequest mapping(String type, String source) { if (mappings.containsKey(type)) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } - mappings.put(type, source); + XContentType xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); + mappings.put(type, new Tuple<>(xContentType, new BytesArray(source.getBytes(StandardCharsets.UTF_8)))); + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the content type of the mapping source + */ + public CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) { + if (mappings.containsKey(type)) { + throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); + } + Objects.requireNonNull(xContentType); + mappings.put(type, new Tuple<>(xContentType, source)); return this; } @@ -235,11 +267,7 @@ public CreateIndexRequest mapping(String type, XContentBuilder source) { if (mappings.containsKey(type)) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } - try { - mappings.put(type, source.string()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); - } + mappings.put(type, new Tuple<>(source.contentType(), source.bytes())); return this; } @@ -399,11 +427,8 @@ public CreateIndexRequest source(BytesReference source) { * Sets the settings and mappings as a single source. */ public CreateIndexRequest source(BytesReference source, XContentType xContentType) { - if (xContentType != null) { - source(XContentHelper.convertToMap(source, false).v2()); - } else { - settings(source.utf8ToString()); - } + Objects.requireNonNull(xContentType); + source(XContentHelper.convertToMap(source, false, xContentType).v2()); return this; } @@ -447,7 +472,7 @@ public CreateIndexRequest source(Map source) { return this; } - public Map mappings() { + public Map> mappings() { return this.mappings; } @@ -520,7 +545,16 @@ public void readFrom(StreamInput in) throws IOException { readTimeout(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + final String type = in.readString(); + final BytesReference bytesReference = in.readBytesReference(); + final XContentType xContentType = XContentType.readFrom(in); + mappings.put(type, new Tuple<>(xContentType, bytesReference)); + } else { + final BytesReference bytesReference = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); + final XContentType xContentType = XContentFactory.xContentType(bytesReference); + mappings.put(in.readString(), new Tuple<>(xContentType, bytesReference)); + } } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { @@ -544,9 +578,17 @@ public void writeTo(StreamOutput out) throws IOException { writeSettingsToStream(settings, out); writeTimeout(out); out.writeVInt(mappings.size()); - for (Map.Entry entry : mappings.entrySet()) { + for (Map.Entry> entry : mappings.entrySet()) { out.writeString(entry.getKey()); - out.writeString(entry.getValue()); + final Tuple value = entry.getValue(); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + out.writeBytesReference(value.v2()); + value.v1().writeTo(out); + } else if (value.v1().hasStringRepresentation()) { + out.writeString(value.v2().utf8ToString()); + } else { + throw new IllegalStateException("cannot send [" + value.v1() + "] to an older node"); + } } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index c9ac84c0b62f7..9642311ed59ea 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -77,13 +77,23 @@ public CreateIndexRequestBuilder setSettings(XContentBuilder builder) { } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * A simplified version of settings that takes key value pairs settings. */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 821023bfd1ed8..7af855b0262bf 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; @@ -45,11 +46,13 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -77,7 +80,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest mappings = new HashMap<>(); + private Map> mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -219,12 +222,12 @@ public Settings settings() { * * @param type The mapping type * @param source The mapping source - * @deprecated use {@link #mapping(String, String, XContentType)} + * @deprecated use {@link #mapping(String, BytesReference, XContentType)} */ @Deprecated public PutIndexTemplateRequest mapping(String type, String source) { XContentType xContentType = XContentFactory.xContentType(source); - return mapping(type, source, xContentType); + return mapping(type, new BytesArray(source.getBytes(StandardCharsets.UTF_8)), xContentType); } /** @@ -234,20 +237,9 @@ public PutIndexTemplateRequest mapping(String type, String source) { * @param source The mapping source * @param xContentType The type of content contained within the source */ - public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { - if (xContentType == null) { - throw new IllegalArgumentException("could not determine xcontent type"); - } else if (xContentType == XContentType.JSON) { - mappings.put(type, source); - } else { - try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, source); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { - builder.copyCurrentStructure(parser); - mappings.put(type, builder.string()); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); - } - } + public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); + mappings.put(type, new Tuple<>(xContentType, source)); return this; } @@ -270,11 +262,7 @@ public String cause() { * @param source The mapping source */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { - try { - return mapping(type, source.string(), source.contentType()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); - } + return mapping(type, source.bytes(), source.contentType()); } /** @@ -291,7 +279,7 @@ public PutIndexTemplateRequest mapping(String type, Map source) try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string(), XContentType.JSON); + return mapping(type, builder.bytes(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -306,7 +294,7 @@ public PutIndexTemplateRequest mapping(String type, Object... source) { return this; } - public Map mappings() { + public Map> mappings() { return this.mappings; } @@ -543,12 +531,17 @@ public void readFrom(StreamInput in) throws IOException { int size = in.readVInt(); if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + final String type = in.readString(); + final BytesReference source = in.readBytesReference(); + final XContentType xContentType = XContentType.readFrom(in); + mappings.put(type, new Tuple<>(xContentType, source)); } } else { - // we cannot expect that the string is json from older versions so we may need to convert for (int i = 0; i < size; i++) { - mapping(in.readString(), in.readString()); + final String type = in.readString(); + final BytesReference source = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); + final XContentType xContentType = XContentFactory.xContentType(source); + mappings.put(type, new Tuple<>(xContentType, source)); } } int customSize = in.readVInt(); @@ -578,9 +571,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(create); writeSettingsToStream(settings, out); out.writeVInt(mappings.size()); - for (Map.Entry entry : mappings.entrySet()) { + for (Map.Entry> entry : mappings.entrySet()) { out.writeString(entry.getKey()); - out.writeString(entry.getValue()); + Tuple value = entry.getValue(); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + out.writeBytesReference(value.v2()); + value.v1().writeTo(out); + } else if (value.v1().hasStringRepresentation()) { + out.writeString(value.v2().utf8ToString()); + } else { + throw new IllegalStateException("cannot send [" + value.v1() + "] to an older node"); + } } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index 60f08a13ef0b9..ab104fa6b91ef 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -131,7 +131,7 @@ public PutIndexTemplateRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source - * @deprecated use {@link #addMapping(String, String, XContentType)} + * @deprecated use {@link #addMapping(String, BytesReference, XContentType)} */ @Deprecated public PutIndexTemplateRequestBuilder addMapping(String type, String source) { @@ -146,7 +146,7 @@ public PutIndexTemplateRequestBuilder addMapping(String type, String source) { * @param source The mapping source * @param xContentType The type/format of the source */ - public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) { + public PutIndexTemplateRequestBuilder addMapping(String type, BytesReference source, XContentType xContentType) { request.mapping(type, source, xContentType); return this; } @@ -251,7 +251,7 @@ public PutIndexTemplateRequestBuilder setSource(Map templateSource) { /** * The template source definition. - * @deprecated use {@link #setSource(String, XContentType)} + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ @Deprecated public PutIndexTemplateRequestBuilder setSource(String templateSource) { @@ -262,14 +262,16 @@ public PutIndexTemplateRequestBuilder setSource(String templateSource) { /** * The template source definition. */ - public PutIndexTemplateRequestBuilder setSource(String templateSource, XContentType xContentType) { + public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource, XContentType xContentType) { request.source(templateSource, xContentType); return this; } /** * The template source definition. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) { request.source(templateSource); return this; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 4ba244e107da8..b7d7cccb81853 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; @@ -347,7 +348,7 @@ public Builder putMapping(String mappingType, CompressedXContent mappingSource) return this; } - public Builder putMapping(String mappingType, String mappingSource) throws IOException { + public Builder putMapping(String mappingType, BytesReference mappingSource) throws IOException { mappings.put(mappingType, new CompressedXContent(mappingSource)); return this; } @@ -456,7 +457,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t String mappingType = currentFieldName; Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); - builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string()); + builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).bytes()); } } } else if ("aliases".equals(currentFieldName)) { @@ -480,7 +481,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t Map mapping = parser.mapOrdered(); if (mapping.size() == 1) { String mappingType = mapping.keySet().iterator().next(); - String mappingSource = XContentFactory.jsonBuilder().map(mapping).string(); + BytesReference mappingSource = XContentFactory.jsonBuilder().map(mapping).bytes(); if (mappingSource == null) { // crap, no mapping source, warn? diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 7015695948930..46b06134a480e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,6 +53,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; @@ -61,7 +63,10 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -252,8 +257,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { List templateNames = new ArrayList<>(); - for (Map.Entry entry : request.mappings().entrySet()) { - mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue())); + for (Map.Entry> entry : request.mappings().entrySet()) { + mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue().v2(), + entry.getValue().v1())); } for (Map.Entry entry : request.customs().entrySet()) { @@ -264,11 +270,14 @@ public ClusterState execute(ClusterState currentState) throws Exception { for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); for (ObjectObjectCursor cursor : template.mappings()) { + BytesReference bytesReference = cursor.value.uncompressedReference(); + XContentType xContentType = XContentFactory.xContentType(bytesReference); if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), - MapperService.parseMapping(xContentRegistry, cursor.value.string())); + MapperService.parseMapping(xContentRegistry, bytesReference, xContentType)); } else { - mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, cursor.value.string())); + mappings.put(cursor.key, + MapperService.parseMapping(xContentRegistry, bytesReference, xContentType)); } } // handle custom diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index e35fc7c837c46..a5bd516d621ee 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -30,6 +30,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; @@ -37,6 +39,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperParsingException; @@ -222,13 +225,14 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla templateBuilder.settings(request.settings); Map> mappingsForValidation = new HashMap<>(); - for (Map.Entry entry : request.mappings.entrySet()) { + for (Map.Entry> entry : request.mappings.entrySet()) { try { - templateBuilder.putMapping(entry.getKey(), entry.getValue()); + templateBuilder.putMapping(entry.getKey(), entry.getValue().v2()); } catch (Exception e) { throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage()); } - mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue())); + mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue().v2(), + entry.getValue().v1())); } dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false); @@ -316,7 +320,7 @@ public static class PutRequest { Integer version; List indexPatterns; Settings settings = Settings.Builder.EMPTY_SETTINGS; - Map mappings = new HashMap<>(); + Map> mappings = new HashMap<>(); List aliases = new ArrayList<>(); Map customs = new HashMap<>(); @@ -347,7 +351,7 @@ public PutRequest settings(Settings settings) { return this; } - public PutRequest mappings(Map mappings) { + public PutRequest mappings(Map> mappings) { this.mappings.putAll(mappings); return this; } @@ -362,8 +366,8 @@ public PutRequest customs(Map customs) { return this; } - public PutRequest putMapping(String mappingType, String mappingSource) { - mappings.put(mappingType, mappingSource); + public PutRequest putMapping(String mappingType, BytesReference mappingSource, XContentType xContentType) { + mappings.put(mappingType, new Tuple<>(xContentType, mappingSource)); return this; } diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java index 991dfccf7f994..059572848dc57 100644 --- a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java +++ b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java @@ -146,6 +146,15 @@ public byte[] uncompressed() { } } + /** Return the uncompressed bytes as a {@link BytesReference} */ + public BytesReference uncompressedReference() { + try { + return CompressorFactory.uncompress(new BytesArray(bytes)); + } catch (IOException e) { + throw new IllegalStateException("Cannot decompress compressed bytes", e); + } + } + public String string() throws IOException { return new BytesRef(uncompressed()).utf8ToString(); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 40b2d980226f4..b8ca85cd1f168 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; @@ -190,8 +191,9 @@ public DocumentMapperParser documentMapperParser() { return this.documentParser; } - public static Map parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception { - try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(xContentRegistry, mappingSource)) { + public static Map parseMapping(NamedXContentRegistry xContentRegistry, BytesReference mappingSource, + XContentType xContentType) throws Exception { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, mappingSource)) { return parser.map(); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java index c3b2eb80584b5..1ae60f42eb63d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java @@ -86,7 +86,7 @@ public void testSetSettings() throws IOException { builder.setSettings(KEY, VALUE); assertEquals(VALUE, builder.request().settings().get(KEY)); - builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}"); + builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON); assertEquals(VALUE, builder.request().settings().get(KEY)); builder.setSettings(Settings.builder().put(KEY, VALUE)); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java index d098dd6befaa2..3615ea10c361c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESSingleNodeTestCase; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; /** * Rudimentary tests that the templates used by Logstash and Beats @@ -30,10 +30,10 @@ */ public class BWCTemplateTests extends ESSingleNodeTestCase { public void testBeatsTemplatesBWC() throws Exception { - String metricBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json"); - String packetBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json"); - String fileBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json"); - String winLogBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json"); + byte[] metricBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json"); + byte[] packetBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json"); + byte[] fileBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json"); + byte[] winLogBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json"); client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat, XContentType.JSON).get(); client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get(); client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get(); @@ -47,7 +47,7 @@ public void testBeatsTemplatesBWC() throws Exception { } public void testLogstashTemplatesBWC() throws Exception { - String ls5x = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json"); + byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json"); client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get(); client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get(); assertWarnings("Deprecated field [template] used, replaced by [index_patterns]"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index 48598ecb2ecdb..5643e4ea27814 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -26,10 +26,12 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService; import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; @@ -100,7 +102,7 @@ public void testIndexTemplateWithAliasNameEqualToTemplatePattern() { public void testIndexTemplateWithValidateEmptyMapping() throws Exception { PutRequest request = new PutRequest("api", "validate_template"); request.patterns(Collections.singletonList("validate_template")); - request.putMapping("type1", "{}"); + request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -113,7 +115,7 @@ public void testIndexTemplateWithValidateMapping() throws Exception { request.patterns(Collections.singletonList("te*")); request.putMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("field2").field("type", "text").field("analyzer", "custom_1").endObject() - .endObject().endObject().endObject().string()); + .endObject().endObject().endObject().bytes(), XContentType.JSON); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -124,7 +126,7 @@ public void testIndexTemplateWithValidateMapping() throws Exception { public void testBrokenMapping() throws Exception { PutRequest request = new PutRequest("api", "broken_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", "abcde"); + request.putMapping("type1", new BytesArray("abcde"), XContentType.JSON); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -135,7 +137,7 @@ public void testBrokenMapping() throws Exception { public void testBlankMapping() throws Exception { PutRequest request = new PutRequest("api", "blank_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", "{}"); + request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -147,7 +149,7 @@ public void testAliasInvalidFilterInvalidJson() throws Exception { //invalid json: put index template fails PutRequest request = new PutRequest("api", "blank_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", "{}"); + request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); Set aliases = new HashSet<>(); aliases.add(new Alias("invalid_alias").filter("abcde")); request.aliases(aliases); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index 00769f692e559..b8dc16274ec04 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; @@ -70,11 +71,11 @@ public void testPutIndexTemplateRequest510() throws IOException { public void testPutIndexTemplateRequestSerializationXContent() throws IOException { PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); - String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + BytesReference mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); request.patterns(Collections.singletonList("foo")); // THIS IS NOT A BUG! Intentionally specifying the wrong type so we serialize it request.mapping("bar", mapping, XContentType.JSON); - assertEquals(mapping, request.mappings().get("bar")); + assertEquals(mapping, request.mappings().get("bar").v2()); BytesStreamOutput out = new BytesStreamOutput(); out.setVersion(Version.V_5_0_0); @@ -84,23 +85,15 @@ public void testPutIndexTemplateRequestSerializationXContent() throws IOExceptio in.setVersion(Version.V_5_0_0); PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); serialized.readFrom(in); - assertNotEquals(mapping, serialized.mappings().get("bar")); - assertTrue(serialized.mappings().get("bar").startsWith("{")); + assertEquals(mapping, serialized.mappings().get("bar").v2()); + assertEquals(XContentType.YAML, serialized.mappings().get("bar").v1()); out = new BytesStreamOutput(); request.writeTo(out); in = StreamInput.wrap(out.bytes().toBytesRef().bytes); serialized = new PutIndexTemplateRequest(); serialized.readFrom(in); - assertEquals(mapping, serialized.mappings().get("bar")); - } - - public void testContentTypesAreConvertedWhenSettingMapping() throws IOException { - PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); - String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); - request.mapping("bar", mapping, XContentType.YAML); - assertNotEquals(mapping, request.mappings().get("bar")); - assertFalse(mapping.startsWith("{")); - assertTrue(request.mappings().get("bar").startsWith("{")); + assertEquals(mapping, serialized.mappings().get("bar").v2()); + assertEquals(XContentType.JSON, serialized.mappings().get("bar").v1()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index f6fd7bbd0784a..c907202dbd9c8 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; @@ -175,13 +176,14 @@ public void testMergeWithMap() throws Throwable { MapperService mapperService = indexService1.mapperService(); Map> mappings = new HashMap<>(); - mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping(xContentRegistry(), "{}")); + mappings.put(MapperService.DEFAULT_MAPPING, + MapperService.parseMapping(xContentRegistry(), new BytesArray("{}"), XContentType.JSON)); MapperException e = expectThrows(MapperParsingException.class, () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false)); assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: ")); mappings.clear(); - mappings.put("type1", MapperService.parseMapping(xContentRegistry(), "{}")); + mappings.put("type1", MapperService.parseMapping(xContentRegistry(), new BytesArray("{}"), XContentType.JSON)); e = expectThrows( MapperParsingException.class, () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false)); diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index e5e5f7c456b15..2d82b1ec025ac 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -336,7 +337,7 @@ public void testBrokenMapping() throws Exception { MapperParsingException e = expectThrows( MapperParsingException.class, () -> client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "abcde", XContentType.JSON) + .addMapping("type1", new BytesArray("abcde"), XContentType.JSON) .get()); assertThat(e.getMessage(), containsString("Failed to parse mapping ")); @@ -373,7 +374,8 @@ public void testIndexTemplateWithAliases() throws Exception { client().admin().indices().preparePutTemplate("template_with_aliases") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON) + .addMapping("type1", new BytesArray("{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}"), + XContentType.JSON) .addAlias(new Alias("simple_alias")) .addAlias(new Alias("templated_alias-{index}")) .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}")) @@ -424,7 +426,7 @@ public void testIndexTemplateWithAliases() throws Exception { public void testIndexTemplateWithAliasesInSource() { client().admin().indices().preparePutTemplate("template_1") - .setSource("{\n" + + .setSource(new BytesArray("{\n" + " \"template\" : \"*\",\n" + " \"aliases\" : {\n" + " \"my_alias\" : {\n" + @@ -435,7 +437,7 @@ public void testIndexTemplateWithAliasesInSource() { " }\n" + " }\n" + " }\n" + - "}", XContentType.JSON).get(); + "}"), XContentType.JSON).get(); assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2")); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index c8cd235aedec2..4a8c88b3093fe 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardException; @@ -305,7 +306,7 @@ public void testXContentResponse() throws Exception { public void testDeletesIssue7951() throws Exception { String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; - assertAcked(prepareCreate(INDEX_NAME).setSettings(settings) + assertAcked(prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON) .addMapping("doc", "text", "type=keyword", CLASS_FIELD, "type=keyword")); String[] cat1v1 = {"constant", "one"}; String[] cat1v2 = {"constant", "uno"}; diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 2914755ed73c7..25e92bc525cd9 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -98,7 +99,7 @@ public void testNoPreferenceRandom() throws Exception { } public void testSimplePreference() throws Exception { - client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}").get(); + client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen(); client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); @@ -131,7 +132,7 @@ public void testSimplePreference() throws Exception { } public void testReplicaPreference() throws Exception { - client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}").get(); + client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}", XContentType.JSON).get(); ensureGreen(); client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 69e4899a73d1c..7cb9627cf1835 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1797,7 +1797,7 @@ public void testRangeQueryWithTimeZone() throws Exception { } public void testSearchEmptyDoc() { - assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}")); + assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}", XContentType.JSON)); client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON).get(); refresh(); diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 7fc1490de73f6..6dd4fa384e99b 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; @@ -80,7 +81,7 @@ public static void index01Docs(String type, String settings, ESIntegTestCase tes if (type.equals("text")) { textMappings += ",fielddata=true"; } - assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings) + assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON) .addMapping("doc", "text", textMappings, CLASS_FIELD, "type=keyword")); String[] gb = {"0", "1"}; List indexRequestBuilderList = new ArrayList<>(); From 7a267ffed928ed3ca011431656aadacd80459df8 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 20 Jan 2017 10:41:01 -0500 Subject: [PATCH 04/34] use Objects.requireNonNull for pipeline requests --- .../action/ingest/PutPipelineRequest.java | 29 +++---------------- .../ingest/SimulatePipelineRequest.java | 24 +++++++-------- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 03bce969026d5..d595edcf4dd1f 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -31,8 +31,6 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.action.ValidateActions.addValidationError; - public class PutPipelineRequest extends AcknowledgedRequest { private String id; @@ -41,32 +39,13 @@ public class PutPipelineRequest extends AcknowledgedRequest @Deprecated public PutPipelineRequest(String id, BytesReference source) { - if (id == null) { - throw new IllegalArgumentException("id is missing"); - } - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - - this.id = id; - this.source = source; - this.xContentType = XContentFactory.xContentType(source); + this(id, source, XContentFactory.xContentType(source)); } public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { - if (id == null) { - throw new IllegalArgumentException("id is missing"); - } - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - if (xContentType == null) { - throw new IllegalArgumentException("unable to determine source type"); - } - - this.id = id; - this.source = source; - this.xContentType = xContentType; + this.id = Objects.requireNonNull(id); + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } PutPipelineRequest() { diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 7dbac307fc6dd..e22aa71c8bfd5 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -37,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.ingest.IngestDocument.MetaData; @@ -47,24 +48,21 @@ public class SimulatePipelineRequest extends ActionRequest { private BytesReference source; private XContentType xContentType = XContentType.JSON; + /** + * Create a new request + * @deprecated use {@link #SimulatePipelineRequest(BytesReference, XContentType)} that does not attempt content autodetection + */ @Deprecated public SimulatePipelineRequest(BytesReference source) { - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - this.source = source; - this.xContentType = XContentFactory.xContentType(source); + this(source, XContentFactory.xContentType(source)); } + /** + * Creates a new request with the given source and its content type + */ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - if (xContentType == null) { - throw new IllegalArgumentException("content type is missing"); - } - this.source = source; - this.xContentType = xContentType; + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } SimulatePipelineRequest() { From 3e78549ed00ba6936e2c8f74cbaa6f6e17aac439 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 20 Jan 2017 15:11:41 -0500 Subject: [PATCH 05/34] deprecate loadFromSource without xcontenttype --- .../put/PutRepositoryRequest.java | 16 +++++++++- .../put/PutRepositoryRequestBuilder.java | 17 +++++++++- .../ClusterUpdateSettingsRequest.java | 24 ++++++++++++-- .../ClusterUpdateSettingsRequestBuilder.java | 21 +++++++++++++ .../create/CreateSnapshotRequest.java | 20 ++++++++++-- .../create/CreateSnapshotRequestBuilder.java | 17 ++++++++++ .../restore/RestoreSnapshotRequest.java | 30 ++++++++++++++++-- .../RestoreSnapshotRequestBuilder.java | 31 +++++++++++++++++++ .../admin/indices/analyze/AnalyzeRequest.java | 2 +- .../settings/put/UpdateSettingsRequest.java | 14 +++++++-- .../put/UpdateSettingsRequestBuilder.java | 15 +++++++-- .../common/settings/Settings.java | 2 ++ .../common/settings/SettingsFilterTests.java | 2 +- .../loader/JsonSettingsLoaderTests.java | 4 ++- .../loader/YamlSettingsLoaderTests.java | 3 +- .../indices/analysis/AnalysisModuleTests.java | 4 ++- .../search/preference/SearchPreferenceIT.java | 2 +- .../AzureStorageSettingsFilterTests.java | 2 +- 18 files changed, 207 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index a06175a598bf2..f37a5d0130c75 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -144,12 +144,26 @@ public PutRepositoryRequest settings(Settings.Builder settings) { * * @param source repository settings in json, yaml or properties format * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content auto-detection */ + @Deprecated public PutRepositoryRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the repository settings. + * + * @param source repository settings in json or yaml format + * @param xContentType the content type of the source + * @return this request + */ + public PutRepositoryRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the repository settings. * @@ -160,7 +174,7 @@ public PutRepositoryRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 39cfa6af7f750..aed09daff2a86 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -89,16 +90,30 @@ public PutRepositoryRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets the repository settings in Json, Yaml or properties format + * Sets the repository settings in Json or Yaml format * * @param source repository settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection */ + @Deprecated public PutRepositoryRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets the repository settings in Json or Yaml format + * + * @param source repository settings + * @param xContentType the contenty type of the source + * @return this builder + */ + public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets the repository settings * diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index 16310b58cbc5b..aa1ecb9f23210 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -83,12 +83,22 @@ public ClusterUpdateSettingsRequest transientSettings(Settings.Builder settings) /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest transientSettings(String source) { this.transientSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) { + this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ @@ -97,7 +107,7 @@ public ClusterUpdateSettingsRequest transientSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - transientSettings(builder.string()); + transientSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -122,12 +132,22 @@ public ClusterUpdateSettingsRequest persistentSettings(Settings.Builder settings /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest persistentSettings(String source) { this.persistentSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) { + this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ @@ -136,7 +156,7 @@ public ClusterUpdateSettingsRequest persistentSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - persistentSettings(builder.string()); + persistentSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index f0492edfeb19f..906b1867b1f09 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -52,12 +53,22 @@ public ClusterUpdateSettingsRequestBuilder setTransientSettings(Settings.Builder /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) { request.transientSettings(settings); return this; } + /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) { + request.transientSettings(settings, xContentType); + return this; + } + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ @@ -84,12 +95,22 @@ public ClusterUpdateSettingsRequestBuilder setPersistentSettings(Settings.Builde /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) { request.persistentSettings(settings); return this; } + /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) { + request.persistentSettings(settings, xContentType); + return this; + } + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index ae715050e80b5..3267b6d9c96ed 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -288,18 +288,34 @@ public CreateSnapshotRequest settings(Settings.Builder settings) { } /** - * Sets repository-specific snapshot settings in JSON, YAML or properties format + * Sets repository-specific snapshot settings in JSON or YAML format *

* See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets repository-specific snapshot settings in JSON or YAML format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public CreateSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets repository-specific snapshot settings. *

@@ -312,7 +328,7 @@ public CreateSnapshotRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index ebdd206b5c3e8..d3b5e12351c28 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -147,12 +148,28 @@ public CreateSnapshotRequestBuilder setSettings(Settings.Builder settings) { * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets repository-specific snapshot settings in YAML or JSON format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets repository-specific snapshot settings. *

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index a7bbd02ee54eb..67ffa9a716893 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -319,12 +319,28 @@ public RestoreSnapshotRequest settings(Settings.Builder settings) { * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets repository-specific restore settings in JSON, YAML or properties format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public RestoreSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets repository-specific restore settings *

@@ -337,7 +353,7 @@ public RestoreSnapshotRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -436,12 +452,22 @@ public RestoreSnapshotRequest indexSettings(Settings.Builder settings) { /** * Sets settings that should be added/changed in all restored indices + * @deprecated use {@link #indexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest indexSettings(String source) { this.indexSettings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets settings that should be added/changed in all restored indices + */ + public RestoreSnapshotRequest indexSettings(String source, XContentType xContentType) { + this.indexSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Sets settings that should be added/changed in all restored indices */ @@ -449,7 +475,7 @@ public RestoreSnapshotRequest indexSettings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - indexSettings(builder.string()); + indexSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 661a1a1d018af..4b220ab677392 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.List; import java.util.Map; @@ -159,12 +160,28 @@ public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) { * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } + /** + * Sets repository-specific restore settings in JSON, YAML or properties format + *

+ * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + /** * Sets repository-specific restore settings *

@@ -251,12 +268,26 @@ public RestoreSnapshotRequestBuilder setIndexSettings(Settings.Builder settings) * * @param source index settings * @return this builder + * @deprecated use {@link #setIndexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setIndexSettings(String source) { request.indexSettings(source); return this; } + /** + * Sets index settings that should be added or replaced during restore + * + * @param source index settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setIndexSettings(String source, XContentType xContentType) { + request.indexSettings(source, xContentType); + return this; + } + /** * Sets index settings that should be added or replaced during restore * diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 6d0824eeb31c5..08f220e0199d8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -75,7 +75,7 @@ public static class NameOrDefinition implements Writeable { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(definition); - this.definition = Settings.builder().loadFromSource(builder.string()).build(); + this.definition = Settings.builder().loadFromSource(builder.string(), builder.contentType()).build(); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse [" + definition + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index 494b9df7bd37b..df1d4d9f46fc2 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -121,13 +121,23 @@ public UpdateSettingsRequest settings(Settings.Builder settings) { } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public UpdateSettingsRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } + /** + * Sets the settings to be updated (either json or yaml format) + */ + public UpdateSettingsRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + /** * Returns true iff the settings update should only add but not update settings. If the setting already exists * it should not be overwritten by this update. The default is false @@ -153,7 +163,7 @@ public UpdateSettingsRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java index 36dfbf3b2d49b..a9cecbfc5a434 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -70,15 +71,25 @@ public UpdateSettingsRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public UpdateSettingsRequestBuilder setSettings(String source) { request.settings(source); return this; } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + */ + public UpdateSettingsRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** + * Sets the settings to be updated */ public UpdateSettingsRequestBuilder setSettings(Map source) { request.settings(source); diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index a3011284d25dd..05d0c46e5f382 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -951,7 +951,9 @@ public Builder put(Dictionary properties) { /** * Loads settings from the actual string content that represents them using the * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * @deprecated use {@link #loadFromSource(String, XContentType)} to avoid content type detection */ + @Deprecated public Builder loadFromSource(String source) { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromSource(source); try { diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java index 14e93a3bbc35b..93f745d290e5c 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java @@ -118,7 +118,7 @@ private void testFiltering(Settings source, Settings filtered, String... pattern source.toXContent(xContentBuilder, request); xContentBuilder.endObject(); String filteredSettingsString = xContentBuilder.string(); - filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build(); + filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build(); assertThat(filteredSettings.getAsMap().entrySet(), equalTo(filtered.getAsMap().entrySet())); } } diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java index d917c0d12c074..fc1300d94138e 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.CoreMatchers.containsString; @@ -52,7 +53,8 @@ public void testDuplicateKeysThrowsException() { assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", XContent.isStrictDuplicateDetectionEnabled()); final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}"; - final SettingsException e = expectThrows(SettingsException.class, () -> Settings.builder().loadFromSource(json).build()); + final SettingsException e = expectThrows(SettingsException.class, + () -> Settings.builder().loadFromSource(json, XContentType.JSON).build()); assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); assertThat( e.toString(), diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java index eeb2df7e1d68c..e4b4de0ceb616 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.nio.charset.StandardCharsets; @@ -74,7 +75,7 @@ public void testDuplicateKeysThrowsException() { String yaml = "foo: bar\nfoo: baz"; SettingsException e = expectThrows(SettingsException.class, () -> { - Settings.builder().loadFromSource(yaml); + Settings.builder().loadFromSource(yaml, XContentType.YAML); }); assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); String msg = e.getCause().getMessage(); diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 012fae17f5c8d..61d325a64e8f7 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.Analysis; @@ -319,7 +320,8 @@ public void testWordListPath() throws Exception { String[] words = new String[]{"donau", "dampf", "schiff", "spargel", "creme", "suppe"}; Path wordListFile = generateWordList(words); - settings = Settings.builder().loadFromSource("index: \n word_list_path: " + wordListFile.toAbsolutePath()).build(); + settings = Settings.builder().loadFromSource("index: \n word_list_path: " + wordListFile.toAbsolutePath(), XContentType.YAML) + .build(); Set wordList = Analysis.getWordSet(env, Version.CURRENT, settings, "index.word_list"); MatcherAssert.assertThat(wordList.size(), equalTo(6)); diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 25e92bc525cd9..4b3b890a20d70 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -148,7 +148,7 @@ public void testReplicaPreference() throws Exception { SearchResponse resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet(); assertThat(resp.getHits().totalHits(), equalTo(1L)); - client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}").get(); + client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen("test"); resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java index e82ed3a18755d..b67073d7d389f 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java @@ -62,7 +62,7 @@ public void testSettingsFiltering() throws IOException { settings.toXContent(xContentBuilder, request); xContentBuilder.endObject(); String filteredSettingsString = xContentBuilder.string(); - filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build(); + filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build(); assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default")); } From 63ab38a0c6805cc5b1e4ac63f6d8ab2722d810b5 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 20 Jan 2017 15:30:20 -0500 Subject: [PATCH 06/34] more deprecations :) --- .../java/org/elasticsearch/common/settings/Settings.java | 5 +++-- .../common/settings/loader/SettingsLoaderFactory.java | 5 ++++- .../org/elasticsearch/common/xcontent/XContentBuilder.java | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index 05d0c46e5f382..c03ae278ab17a 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -968,6 +968,7 @@ public Builder loadFromSource(String source) { /** * Loads settings from the actual string content that represents them using the * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader + * Note only types that return {@code true} from {@link XContentType#hasStringRepresentation()} are supported */ public Builder loadFromSource(String source, XContentType xContentType) { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType); @@ -982,7 +983,7 @@ public Builder loadFromSource(String source, XContentType xContentType) { /** * Loads settings from a url that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromPath(Path path) throws IOException { // NOTE: loadFromStream will close the input stream @@ -991,7 +992,7 @@ public Builder loadFromPath(Path path) throws IOException { /** * Loads settings from a stream that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromStream(String resourceName, InputStream is) throws IOException { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName); diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java index d454ca380e87a..1e4ae185115e2 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java +++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java @@ -65,7 +65,9 @@ public static SettingsLoader loaderFromResource(String resourceName) { * * @param source The underlying settings content. * @return A settings loader. + * @deprecated use {@link #loaderFromXContentType(XContentType)} instead */ + @Deprecated public static SettingsLoader loaderFromSource(String source) { if (source.indexOf('{') != -1 && source.indexOf('}') != -1) { return new JsonSettingsLoader(true); @@ -77,7 +79,8 @@ public static SettingsLoader loaderFromSource(String source) { } /** - * Returns a {@link SettingsLoader} based on the {@link XContentType} + * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only types that return {@code true} from + * {@link XContentType#hasStringRepresentation()} are supported * * @param xContentType The content type * @return A settings loader. diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index f9e14acdac1a4..a64310375107e 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -964,6 +964,7 @@ public XContentBuilder byteSizeField(String rawFieldName, String readableFieldNa // Raw fields ////////////////////////////////// + @Deprecated public XContentBuilder rawField(String name, InputStream value) throws IOException { generator.writeRawField(name, value); return this; @@ -974,6 +975,7 @@ public XContentBuilder rawField(String name, InputStream value, XContentType con return this; } + @Deprecated public XContentBuilder rawField(String name, BytesReference value) throws IOException { generator.writeRawField(name, value); return this; @@ -984,6 +986,7 @@ public XContentBuilder rawField(String name, BytesReference value, XContentType return this; } + @Deprecated public XContentBuilder rawValue(BytesReference value) throws IOException { generator.writeRawValue(value); return this; From 99f551ffa28268b3f4241bee23d9e4b49853b59e Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 23 Jan 2017 13:02:21 -0500 Subject: [PATCH 07/34] cleanup deprecated usages and remove source methods without xcontenttype --- .../mapping/get/GetFieldMappingsResponse.java | 27 ++++++++--- .../TransportGetFieldMappingsIndexAction.java | 4 +- .../action/bulk/TransportShardBulkAction.java | 10 +++-- .../termvectors/TermVectorsRequest.java | 32 ++++++++++++- .../common/xcontent/XContentHelper.java | 37 +++++++++++---- .../common/xcontent/XContentType.java | 20 +++++++-- .../index/mapper/SourceFieldMapper.java | 4 +- .../index/mapper/SourceToParse.java | 12 ++--- .../index/query/MoreLikeThisQueryBuilder.java | 26 ++++++++--- .../shard/TranslogRecoveryPerformer.java | 4 +- .../index/termvectors/TermVectorsService.java | 12 +++-- .../node/tasks/TransportTasksActionTests.java | 2 +- .../ingest/WriteableIngestDocumentTests.java | 2 +- .../support/XContentMapValuesTests.java | 10 ++--- .../index/mapper/DynamicMappingTests.java | 4 +- .../index/mapper/IdFieldMapperTests.java | 5 ++- .../index/mapper/MultiFieldTests.java | 3 +- .../index/mapper/ParentFieldMapperTests.java | 10 +++-- .../index/mapper/RoutingFieldMapperTests.java | 3 +- .../builder/SearchSourceBuilderTests.java | 4 +- .../search/scroll/SearchScrollIT.java | 2 +- .../snapshots/SnapshotRequestsTests.java | 4 +- .../ThreadPoolSerializationTests.java | 4 +- .../script/mustache/MustacheTests.java | 6 ++- .../percolator/PercolateQueryBuilder.java | 29 ++++++++++-- .../PercolateQueryBuilderTests.java | 14 +++--- .../PercolatorFieldMapperTests.java | 4 +- .../percolator/PercolatorQuerySearchIT.java | 45 ++++++++++--------- .../index/mapper/size/SizeMappingTests.java | 7 +-- .../index/shard/IndexShardTestCase.java | 11 ++++- .../elasticsearch/test/XContentTestUtils.java | 2 +- 31 files changed, 249 insertions(+), 110 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index e0cedcf841e47..ede90ba5f3528 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.mapping.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -26,7 +27,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; import java.io.IOException; @@ -92,14 +95,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static class FieldMappingMetaData implements ToXContent { - public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY); + public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY, XContentType.JSON); private String fullName; private BytesReference source; + private XContentType xContentType; - public FieldMappingMetaData(String fullName, BytesReference source) { + public FieldMappingMetaData(String fullName, BytesReference source, XContentType xContentType) { this.fullName = fullName; this.source = source; + this.xContentType = xContentType; } public String fullName() { @@ -108,7 +113,7 @@ public String fullName() { /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */ public Map sourceAsMap() { - return XContentHelper.convertToMap(source, true).v2(); + return XContentHelper.convertToMap(source, true, xContentType).v2(); } public boolean isNull() { @@ -121,7 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (params.paramAsBoolean("pretty", false)) { builder.field("mapping", sourceAsMap()); } else { - builder.rawField("mapping", source); + builder.rawField("mapping", source, xContentType); } return builder; } @@ -141,7 +146,16 @@ public void readFrom(StreamInput in) throws IOException { int fieldSize = in.readVInt(); Map fieldMapBuilder = new HashMap<>(fieldSize); for (int k = 0; k < fieldSize; k++) { - fieldMapBuilder.put(in.readString(), new FieldMappingMetaData(in.readString(), in.readBytesReference())); + final String key = in.readString(); + final String field = in.readString(); + final BytesReference bytesReference = in.readBytesReference(); + final XContentType xContentType; + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(bytesReference); + } + fieldMapBuilder.put(key, new FieldMappingMetaData(field, bytesReference, xContentType)); } typeMapBuilder.put(type, unmodifiableMap(fieldMapBuilder)); } @@ -165,6 +179,9 @@ public void writeTo(StreamOutput out) throws IOException { FieldMappingMetaData fieldMapping = fieldEntry.getValue(); out.writeString(fieldMapping.fullName()); out.writeBytesReference(fieldMapping.source); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + fieldMapping.xContentType.writeTo(out); + } } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 864c6703c48e0..a26e864ba53e7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -214,10 +214,10 @@ private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuilder i private BytesReference doc; + private XContentType xContentType; + private String routing; private String parent; @@ -156,8 +161,9 @@ public TermVectorsRequest(TermVectorsRequest other) { super(other.index()); this.id = other.id(); this.type = other.type(); - if (this.doc != null) { + if (other.doc != null) { this.doc = new BytesArray(other.doc().toBytesRef(), true); + this.xContentType = other.xContentType; } this.flagsEnum = other.getFlags().clone(); this.preference = other.preference(); @@ -225,22 +231,36 @@ public BytesReference doc() { return doc; } + public XContentType xContentType() { + return xContentType; + } + /** * Sets an artificial document from which term vectors are requested for. */ public TermVectorsRequest doc(XContentBuilder documentBuilder) { - return this.doc(documentBuilder.bytes(), true); + return this.doc(documentBuilder.bytes(), true, documentBuilder.contentType()); } /** * Sets an artificial document from which term vectors are requested for. + * @deprecated use {@link #doc(BytesReference, boolean, XContentType)} to avoid content auto detection */ + @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { + return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc)); + } + + /** + * Sets an artificial document from which term vectors are requested for. + */ + public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, XContentType xContentType) { // assign a random id to this artificial document, for routing if (generateRandomId) { this.id(String.valueOf(randomInt.getAndAdd(1))); } this.doc = doc; + this.xContentType = xContentType; return this; } @@ -479,6 +499,11 @@ public void readFrom(StreamInput in) throws IOException { if (in.readBoolean()) { doc = in.readBytesReference(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } routing = in.readOptionalString(); parent = in.readOptionalString(); @@ -519,6 +544,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } out.writeOptionalString(routing); out.writeOptionalString(parent); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index ea88828666ac5..dbbfa0889cd60 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -55,16 +55,41 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry } } + /** + * Converts the given bytes into a map that is optionally ordered. + * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)} + * instead with the proper {@link XContentType} + */ @Deprecated public static Tuple> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { - return convertToMap(bytes, ordered, null); + try { + final XContentType contentType; + InputStream input; + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + InputStream compressedStreamInput = compressor.streamInput(bytes.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } + contentType = XContentFactory.xContentType(compressedStreamInput); + input = compressedStreamInput; + } else { + contentType = XContentFactory.xContentType(bytes); + input = bytes.streamInput(); + } + return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse content to map", e); + } } + /** + * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null. + */ public static Tuple> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) throws ElasticsearchParseException { try { - XContentType contentType = xContentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -72,17 +97,11 @@ public static Tuple> convertToMap(BytesReferen if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); } - if (contentType == null) { - contentType = XContentFactory.xContentType(compressedStreamInput); - } input = compressedStreamInput; } else { - if (contentType == null) { - contentType = XContentFactory.xContentType(bytes); - } input = bytes.streamInput(); } - return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); + return new Tuple<>(Objects.requireNonNull(xContentType), convertToMap(XContentFactory.xContent(xContentType), input, ordered)); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 09cf5a6f2ac6a..ad3acfa4e8c6a 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -137,12 +137,18 @@ public boolean hasStringRepresentation() { } }; + /** + * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has + * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method + * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a + * format query parameter. This method will return {@code null} if no match is found + */ public static XContentType fromMediaTypeOrFormat(String mediaType) { if (mediaType == null) { return null; } for (XContentType type : values()) { - if (isSameMediaTypeAs(mediaType, type)) { + if (isSameMediaTypeOrFormatAs(mediaType, type)) { return type; } } @@ -153,17 +159,22 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) { return null; } + /** + * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. + * The provided media type should not included any parameters. This method is suitable for parsing part of the {@code Content-Type} + * HTTP header. This method will return {@code null} if no match is found + */ public static XContentType fromMediaTypeStrict(String mediaType) { final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT); for (XContentType type : values()) { - if (type.mediaTypeWithoutParameters().equals(mediaType)) { + if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) { return type; } } return null; } - private static boolean isSameMediaTypeAs(String stringType, XContentType type) { + private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) { return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) || stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") || type.shortName().equalsIgnoreCase(stringType); @@ -189,6 +200,9 @@ public String mediaType() { public abstract String mediaTypeWithoutParameters(); + /** + * Returns {@code true} if this {@link XContentType} can be represented as a String and is not a binary only format + */ public abstract boolean hasStringRepresentation(); public static XContentType readFrom(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index e06ec80a472dd..6a0129af7683e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -241,7 +241,9 @@ protected void parseCreateField(ParseContext context, List field if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(source, true); + Tuple> mapTuple = context.sourceToParse().getXContentType() == null ? + XContentHelper.convertToMap(source, true) : + XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType()); Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); XContentType contentType = mapTuple.v1(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 55682c4acd330..3b8ddb113abd2 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -28,18 +28,10 @@ public class SourceToParse { - public static SourceToParse source(String index, String type, String id, BytesReference source) { - return source(Origin.PRIMARY, index, type, id, source, XContentFactory.xContentType(source)); - } - public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) { return source(Origin.PRIMARY, index, type, id, source, contentType); } - public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) { - return source(origin, index, type, id, source, XContentFactory.xContentType(source)); - } - public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source, XContentType contentType) { return new SourceToParse(origin, index, type, id, source, contentType); @@ -105,6 +97,10 @@ public String routing() { return this.routing; } + public XContentType getXContentType() { + return this.xContentType; + } + public SourceToParse routing(String routing) { this.routing = routing; return this; diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index bfb579e0c322c..ea61edbbf9cad 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; @@ -161,6 +162,7 @@ public interface Field { private String type; private String id; private BytesReference doc; + private XContentType xContentType; private String[] fields; private Map perFieldAnalyzer; private String routing; @@ -178,6 +180,7 @@ public Item() { this.type = copy.type; this.id = copy.id; this.doc = copy.doc; + this.xContentType = copy.xContentType; this.fields = copy.fields; this.perFieldAnalyzer = copy.perFieldAnalyzer; this.version = copy.version; @@ -214,6 +217,7 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) this.index = index; this.type = type; this.doc = doc.bytes(); + this.xContentType = doc.contentType(); } /** @@ -224,6 +228,11 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) type = in.readOptionalString(); if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } else { id = in.readString(); } @@ -241,6 +250,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); + if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } else { out.writeString(id); } @@ -325,6 +337,10 @@ public Item versionType(VersionType versionType) { return this; } + public XContentType xContentType() { + return xContentType; + } + /** * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document. */ @@ -342,7 +358,7 @@ public TermVectorsRequest toTermVectorsRequest() { .termStatistics(false); // for artificial docs to make sure that the id has changed in the item too if (doc != null) { - termVectorsRequest.doc(doc, true); + termVectorsRequest.doc(doc, true, xContentType); this.id = termVectorsRequest.id(); } return termVectorsRequest; @@ -366,6 +382,7 @@ public static Item parse(XContentParser parser, Item item) throws IOException { item.id = parser.text(); } else if (Field.DOC.match(currentFieldName)) { item.doc = jsonBuilder().copyCurrentStructure(parser).bytes(); + item.xContentType = XContentType.JSON; } else if (Field.FIELDS.match(currentFieldName)) { if (token == XContentParser.Token.START_ARRAY) { List fields = new ArrayList<>(); @@ -416,12 +433,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Field.ID.getPreferredName(), this.id); } if (this.doc != null) { - XContentType contentType = XContentFactory.xContentType(this.doc); - if (contentType == builder.contentType()) { - builder.rawField(Field.DOC.getPreferredName(), this.doc); - } else { - builder.rawField(Field.DOC.getPreferredName(), doc); - } + builder.rawField(Field.DOC.getPreferredName(), this.doc, xContentType); } if (this.fields != null) { builder.array(Field.FIELDS.getPreferredName(), this.fields); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index b5c23d9dc467d..0ec2bf975872c 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; @@ -157,7 +158,8 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio Translog.Index index = (Translog.Index) operation; // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all // autoGeneratedID docs that are coming from the primary are updated correctly. - Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source()) + Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), + source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 520cb13390f79..7981c89124280 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetField; @@ -270,7 +271,8 @@ private static Fields generateTermVectors(IndexShard indexShard, Map serialize(ListTasksResponse response, boolean byPare builder.endObject(); builder.flush(); logger.info(builder.string()); - return XContentHelper.convertToMap(builder.bytes(), false).v2(); + return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index bc72094558e7c..d5417526c0cae 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -127,7 +127,7 @@ public void testToXContent() throws IOException { builder.startObject(); writeableIngestDocument.toXContent(builder, EMPTY_PARAMS); builder.endObject(); - Map toXContentMap = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map toXContentMap = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); Map toXContentDoc = (Map) toXContentMap.get("doc"); Map toXContentSource = (Map) toXContentDoc.get("_source"); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index a607b48f79ffd..3a1144afe93e0 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -385,7 +385,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exceptio .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj"}, Strings.EMPTY_ARRAY); assertThat(mapTuple.v2(), equalTo(filteredSource)); @@ -397,7 +397,7 @@ public void testThatFilterIncludesEmptyObjectWhenUsingExcludes() throws Exceptio .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"}); assertThat(mapTuple.v2(), equalTo(filteredSource)); @@ -410,7 +410,7 @@ public void testNotOmittingObjectsWithExcludedProperties() throws Exception { .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"obj.f1"}); assertThat(filteredSource.size(), equalTo(1)); @@ -430,7 +430,7 @@ public void testNotOmittingObjectWithNestedExcludedObject() throws Exception { .endObject(); // implicit include - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"*.obj2"}); assertThat(filteredSource.size(), equalTo(1)); @@ -460,7 +460,7 @@ public void testIncludingObjectWithNestedIncludedObject() throws Exception { .endObject() .endObject(); - Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); + Tuple> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType()); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, Strings.EMPTY_ARRAY); assertThat(filteredSource.size(), equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index f5c8d38503ebb..41256113cd6bf 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -197,7 +197,7 @@ private String serialize(ToXContent mapper) throws Exception { private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes()); + SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes(), builder.contentType()); try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) { ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser); assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); @@ -550,7 +550,7 @@ public void testMixTemplateMultiFieldAndMappingReuse() throws Exception { XContentBuilder json = XContentFactory.jsonBuilder().startObject() .field("field", "foo") .endObject(); - SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes()); + SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes(), json.contentType()); DocumentMapper mapper = indexService.mapperService().documentMapper("type1"); assertNull(mapper.mappers().getMapper("field.raw")); ParsedDocument parsed = mapper.parse(source); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 55b6e7bf71c30..b7ad6a7e4c3de 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; @@ -33,7 +34,7 @@ import static org.hamcrest.Matchers.nullValue; public class IdFieldMapperTests extends ESSingleNodeTestCase { - + public void testId() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); @@ -54,7 +55,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { try { docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder() - .startObject().field("_id", "1").endObject().bytes())); + .startObject().field("_id", "1").endObject().bytes(), XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java index 5c70465d77f5c..8f17b3e0e0d64 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java @@ -186,7 +186,8 @@ public void testMultiFieldsInConsistentOrder() throws Exception { DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); Arrays.sort(multiFieldNames); - Map sourceAsMap = XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true).v2(); + Map sourceAsMap = + XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true, builder.contentType()).v2(); @SuppressWarnings("unchecked") Map multiFields = (Map) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap); assertThat(multiFields.size(), equalTo(multiFieldNames.length)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java index 225940d8eda3e..af5061e62ae95 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -53,7 +54,7 @@ public void testParentSetInDocNotAllowed() throws Exception { try { docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder() - .startObject().field("_parent", "1122").endObject().bytes())); + .startObject().field("_parent", "1122").endObject().bytes(), XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document")); @@ -72,13 +73,14 @@ public void testJoinFieldSet() throws Exception { // Indexing parent doc: DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type"); - ParsedDocument doc = parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"))); + ParsedDocument doc = + parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"), XContentType.JSON)); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); // Indexing child doc: DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type"); - doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122")); + doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}"), XContentType.JSON).parent("1122")); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); @@ -92,7 +94,7 @@ public void testJoinFieldNotSet() throws Exception { .startObject() .field("x_field", "x_value") .endObject() - .bytes())); + .bytes(), XContentType.JSON)); assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc())); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 92e93ede9aa7a..9c26a9806e331 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; @@ -40,7 +41,7 @@ public void testRoutingMapper() throws Exception { .startObject() .field("field", "value") .endObject() - .bytes()).routing("routing_value")); + .bytes(), XContentType.JSON).routing("routing_value")); assertThat(doc.rootDoc().get("_routing"), equalTo("routing_value")); assertThat(doc.rootDoc().get("field"), equalTo("value")); diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index a4096c0e34dbf..ef1750a448c1e 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -280,7 +280,7 @@ public void testToXContent() throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = builder.bytes(); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); + Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); assertEquals(0, sourceAsMap.size()); } { @@ -289,7 +289,7 @@ public void testToXContent() throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = builder.bytes(); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); + Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); assertEquals(1, sourceAsMap.size()); assertEquals("query", sourceAsMap.keySet().iterator().next()); } diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 3927d87912901..1f368cb35daf4 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -521,7 +521,7 @@ public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException { private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); response.toXContent(builder, ToXContent.EMPTY_PARAMS); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map.get("succeeded"), is(succeed)); assertThat(map.get("num_freed"), equalTo(numFreed)); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java index cdacabdd66a14..a8e1eeaec106b 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java @@ -79,7 +79,7 @@ public void testRestoreSnapshotRequestParsing() throws IOException { BytesReference bytes = builder.endObject().bytes(); - request.source(XContentHelper.convertToMap(bytes, true).v2()); + request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2()); assertEquals("test-repo", request.repository()); assertEquals("test-snap", request.snapshot()); @@ -137,7 +137,7 @@ public void testCreateSnapshotRequestParsing() throws IOException { BytesReference bytes = builder.endObject().bytes(); - request.source(XContentHelper.convertToMap(bytes, true).v2()); + request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2()); assertEquals("test-repo", request.repository()); assertEquals("test-snap", request.snapshot()); diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java index 01fc15e6f75f5..33047b2d12a6a 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java @@ -78,7 +78,7 @@ public void testThatToXContentWritesOutUnboundedCorrectly() throws Exception { info.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map, hasKey("foo")); map = (Map) map.get("foo"); assertThat(map, hasKey("queue_size")); @@ -100,7 +100,7 @@ public void testThatToXContentWritesInteger() throws Exception { info.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - Map map = XContentHelper.convertToMap(builder.bytes(), false).v2(); + Map map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); assertThat(map, hasKey("foo")); map = (Map) map.get("foo"); assertThat(map, hasKey("queue_size")); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index ba19febfd214d..2b11df19392a9 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -279,7 +279,8 @@ public void testEmbeddedToJSON() throws Exception { .endArray() .endObject(); - Map ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2()); + Map ctx = + Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2()); assertScript("{{#ctx.bulks}}{{#toJson}}.{{/toJson}}{{/ctx.bulks}}", ctx, equalTo("{\"index\":\"index-1\",\"id\":1,\"type\":\"type-1\"}{\"index\":\"index-2\",\"id\":2,\"type\":\"type-2\"}")); @@ -320,7 +321,8 @@ public void testEmbeddedArrayJoin() throws Exception { .endArray() .endObject(); - Map ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2()); + Map ctx = + Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2()); assertScript("{{#join}}ctx.people.0.emails{{/join}}", ctx, equalTo("john@smith.com,john.smith@email.com,jsmith@email.com")); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 632c50228389f..561a817fc1d48 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -57,6 +57,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.analysis.FieldNameAnalyzer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperForType; @@ -94,6 +95,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder pqb.toQuery(createShardContext())); assertThat(e.getMessage(), equalTo("query builder must be rewritten first")); QueryBuilder rewrite = pqb.rewrite(createShardContext()); - PercolateQueryBuilder geoShapeQueryBuilder = new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource); + PercolateQueryBuilder geoShapeQueryBuilder = + new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource, XContentType.JSON); assertEquals(geoShapeQueryBuilder, rewrite); } @@ -176,14 +178,16 @@ protected Set getObjectsHoldingArbitraryContent() { public void testRequiredParameters() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - new PercolateQueryBuilder(null, null, new BytesArray("{}")); + new PercolateQueryBuilder(null, null, new BytesArray("{}"), XContentType.JSON); }); assertThat(e.getMessage(), equalTo("[field] is a required argument")); - e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"))); + e = expectThrows(IllegalArgumentException.class, + () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"), XContentType.JSON)); assertThat(e.getMessage(), equalTo("[document_type] is a required argument")); - e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", "_document_type", null)); + e = expectThrows(IllegalArgumentException.class, + () -> new PercolateQueryBuilder("_field", "_document_type", null, null)); assertThat(e.getMessage(), equalTo("[document] is a required argument")); e = expectThrows(IllegalArgumentException.class, () -> { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 4d94ab3a29611..d47e0a461443c 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -565,7 +565,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception { ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() - .rawField(fieldName, new BytesArray(query.string())) + .rawField(fieldName, new BytesArray(query.string()), query.contentType()) .endObject().bytes()); BytesRef querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); Map parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); @@ -592,7 +592,7 @@ public void testImplicitlySetDefaultScriptLang() throws Exception { doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() - .rawField(fieldName, new BytesArray(query.string())) + .rawField(fieldName, new BytesArray(query.string()), query.contentType()) .endObject().bytes()); querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 962c453727314..cbbd60f958cb6 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -95,7 +95,7 @@ public void testPercolateScriptQuery() throws IOException { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .execute().actionGet(); SearchResponse response = client().prepareSearch("index") - .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes())) + .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON)) .get(); assertHitCount(response, 1); assertSearchHits(response, "1"); @@ -123,7 +123,7 @@ public void testPercolatorQuery() throws Exception { BytesReference source = jsonBuilder().startObject().endObject().bytes(); logger.info("percolating empty doc"); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -131,7 +131,7 @@ public void testPercolatorQuery() throws Exception { source = jsonBuilder().startObject().field("field1", "value").endObject().bytes(); logger.info("percolating doc with 1 field"); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 2); @@ -141,7 +141,7 @@ public void testPercolatorQuery() throws Exception { source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes(); logger.info("percolating doc with 2 fields"); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 3); @@ -200,7 +200,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test long range: BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("3")); @@ -208,7 +208,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field1", 11).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -216,7 +216,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test double range: source = jsonBuilder().startObject().field("field2", 12).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("6")); @@ -224,7 +224,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field2", 11).endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("4")); @@ -232,7 +232,7 @@ public void testPercolatorRangeQueries() throws Exception { // Test IP range: source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("9")); @@ -240,7 +240,7 @@ public void testPercolatorRangeQueries() throws Exception { source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes(); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("7")); @@ -377,7 +377,7 @@ public void testPercolatorSpecificQueries() throws Exception { .field("field2", "the quick brown fox falls down into the well") .endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", source)) + .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 4); @@ -424,7 +424,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { .field("field1", "The quick brown fox jumps over the lazy dog") .endObject().bytes(); SearchResponse searchResponse = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", "type", document)) + .setQuery(new PercolateQueryBuilder("query", "type", document, XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) .addSort("_uid", SortOrder.ASC) .get(); @@ -458,7 +458,7 @@ public void testTakePositionOffsetGapIntoAccount() throws Exception { client().admin().indices().prepareRefresh().get(); SearchResponse response = client().prepareSearch().setQuery( - new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}")) + new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"), XContentType.JSON) ).get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); @@ -518,7 +518,7 @@ public void testWithMultiplePercolatorFields() throws Exception { BytesReference source = jsonBuilder().startObject().field("field", "value").endObject().bytes(); SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source)) + .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source, XContentType.JSON)) .setIndices("test1") .get(); assertHitCount(response, 1); @@ -527,7 +527,7 @@ public void testWithMultiplePercolatorFields() throws Exception { assertThat(response.getHits().getAt(0).index(), equalTo("test1")); response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source)) + .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source, XContentType.JSON)) .setIndices("test2") .get(); assertHitCount(response, 1); @@ -577,7 +577,7 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { .startObject().field("name", "virginia potts").endObject() .startObject().field("name", "tony stark").endObject() .endArray() - .endObject().bytes())) + .endObject().bytes(), XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 1); @@ -591,14 +591,15 @@ public void testPercolateQueryWithNestedDocuments() throws Exception { .startObject().field("name", "virginia stark").endObject() .startObject().field("name", "tony stark").endObject() .endArray() - .endObject().bytes())) + .endObject().bytes(), XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 0); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "employee", - XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes())) + XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes(), + XContentType.JSON)) .addSort("_doc", SortOrder.ASC) .get(); assertHitCount(response, 0); @@ -633,16 +634,16 @@ public void testPercolatorQueryViaMultiSearch() throws Exception { MultiSearchResponse response = client().prepareMultiSearch() .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - jsonBuilder().startObject().field("field1", "b").endObject().bytes()))) + jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - yamlBuilder().startObject().field("field1", "c").endObject().bytes()))) + yamlBuilder().startObject().field("field1", "c").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - smileBuilder().startObject().field("field1", "b c").endObject().bytes()))) + smileBuilder().startObject().field("field1", "b c").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", - jsonBuilder().startObject().field("field1", "d").endObject().bytes()))) + jsonBuilder().startObject().field("field1", "d").endObject().bytes(), XContentType.JSON))) .add(client().prepareSearch("test") .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null))) .add(client().prepareSearch("test") // non existing doc, so error element diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index 3fa5300fb4383..2cde1b1bd07d2 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -60,7 +61,7 @@ public void testSizeEnabled() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); boolean stored = false; boolean points = false; @@ -81,7 +82,7 @@ public void testSizeDisabled() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); assertThat(doc.rootDoc().getField("_size"), nullValue()); } @@ -95,7 +96,7 @@ public void testSizeNotSet() throws Exception { .field("field", "value") .endObject() .bytes(); - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source)); + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)); assertThat(doc.rootDoc().getField("_size"), nullValue()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 605b9026c2636..8763c2a0c9f5f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MapperTestUtils; @@ -454,17 +455,23 @@ protected Engine.Index indexDoc(IndexShard shard, String type, String id) { } protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source) { + return indexDoc(shard, type, id, source, XContentType.JSON); + } + + protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source, XContentType xContentType) { final Engine.Index index; if (shard.routingEntry().primary()) { index = shard.prepareIndexOnPrimary( - SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)), + SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source), + xContentType), Versions.MATCH_ANY, VersionType.INTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); } else { index = shard.prepareIndexOnReplica( - SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)), + SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source), + xContentType), randomInt(1 << 10), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false); } shard.index(index); diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index 31380e25eb3d4..dcad7187fbe68 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -41,7 +41,7 @@ public static Map convertToMap(ToXContent part) throws IOExcepti builder.startObject(); part.toXContent(builder, EMPTY_PARAMS); builder.endObject(); - return XContentHelper.convertToMap(builder.bytes(), false).v2(); + return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); } From d4f0e13408b99973521965ecb1fa3004b0ba4186 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 23 Jan 2017 14:30:26 -0500 Subject: [PATCH 08/34] more updates and add a sendErrorResponse method --- .../rest/AbstractRestChannel.java | 6 +++ .../org/elasticsearch/rest/RestChannel.java | 4 ++ .../elasticsearch/rest/RestController.java | 43 ++++++++++--------- .../org/elasticsearch/rest/RestRequest.java | 10 ++--- .../rest/RestControllerTests.java | 6 +-- .../test/rest/yaml/ClientYamlTestClient.java | 2 +- 6 files changed, 42 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 73133fe86cb0f..fc6d5f87f0cf8 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -123,4 +123,10 @@ public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } + @Override + public void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException { + sendResponse(new BytesRestResponse(restStatus, newErrorBuilder().startObject() + .field("error", errorMessage) + .endObject())); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index b8408071ef52c..91bbc0e9c6132 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -49,4 +49,8 @@ public interface RestChannel { void sendResponse(RestResponse response); + /** + * Sends a new simple error response with the error message included + */ + void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 73d590c140bf9..ce43f6c9362d5 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -49,6 +49,7 @@ import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.FORBIDDEN; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE; import static org.elasticsearch.rest.RestStatus.OK; public class RestController extends AbstractComponent { @@ -167,7 +168,7 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont RestChannel responseChannel = channel; try { final int contentLength = request.hasContent() ? request.content().length() : 0; - if (checkContentType(request, responseChannel, contentLength)) { + if (checkForContentType(request, responseChannel, contentLength)) { if (canTripCircuitBreaker(request)) { inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); } else { @@ -217,23 +218,24 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final } } - boolean checkContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength) { + /** + * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present and matches an + * {@link XContentType} or the request is plain text. + */ + boolean checkForContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength) { if (contentLength > 0) { if (restRequest.getXContentType() == null && restRequest.isPlainText() == false) { + final List contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); + final String errorMessage; + if (contentTypeHeader == null) { + errorMessage = "Content-Type header is missing"; + } else { + errorMessage = "Content-Type header [" + + Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; + } + try { - XContentBuilder builder = channel.newErrorBuilder(); - final List contentTypeHeader = restRequest.getHeader("Content-Type"); - final String errorMessage; - if (contentTypeHeader == null) { - errorMessage = "Content-Type header is missing"; - } else { - errorMessage = "Content-Type header [" + - Strings.collectionToCommaDelimitedString(restRequest.getHeader("Content-Type")) + "] is not supported"; - } - builder.startObject().field("error", errorMessage).endObject(); - RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); - response.addHeader("Content-Type", builder.contentType().mediaType()); - channel.sendResponse(response); + channel.sendErrorResponse(NOT_ACCEPTABLE, errorMessage); } catch (IOException e) { logger.warn("Failed to send response", e); } @@ -252,11 +254,7 @@ boolean checkRequestParameters(final RestRequest request, final RestChannel chan // we consume the error_trace parameter first to ensure that it is always consumed if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) { try { - XContentBuilder builder = channel.newErrorBuilder(); - builder.startObject().field("error", "error traces in responses are disabled.").endObject(); - RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); - response.addHeader("Content-Type", builder.contentType().mediaType()); - channel.sendResponse(response); + channel.sendErrorResponse(BAD_REQUEST, "error traces in responses are disabled."); } catch (IOException e) { logger.warn("Failed to send response", e); } @@ -366,6 +364,11 @@ public void sendResponse(RestResponse response) { delegate.sendResponse(response); } + @Override + public void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException { + delegate.sendErrorResponse(restStatus, errorMessage); + } + private void close() { // attempt to close once atomically if (closed.compareAndSet(false, true) == false) { diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index b4508f92b4efc..0029a47cb63c4 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -81,7 +81,7 @@ public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map contentType = getHeader("Content-Type"); + final List contentType = getAllHeaderValues("Content-Type"); this.xContentType = parseContentType(contentType); this.isPlainText = xContentType == null && isPlainTextContentType(contentType); } @@ -99,7 +99,7 @@ public RestRequest(NamedXContentRegistry xContentRegistry, Map p this.params = params; this.rawPath = path; this.headers = Collections.unmodifiableMap(headers); - final List contentType = getHeader("Content-Type"); + final List contentType = getAllHeaderValues("Content-Type"); this.xContentType = parseContentType(contentType); this.isPlainText = xContentType == null && isPlainTextContentType(contentType); } @@ -135,7 +135,7 @@ public final String path() { /** * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are - * sent. Use of {@link #getHeader(String)} should be preferred + * sent. Use of {@link #getAllHeaderValues(String)} should be preferred */ public final String header(String name) { List values = headers.get(name); @@ -148,7 +148,7 @@ public final String header(String name) { /** * Get all values for the header or {@code null} if the header was not found */ - public final List getHeader(String name) { + public final List getAllHeaderValues(String name) { List values = headers.get(name); if (values != null) { return Collections.unmodifiableList(values); @@ -401,7 +401,7 @@ public final Tuple contentOrSourceParam() { String source = param("source"); if (source != null) { BytesArray bytes = new BytesArray(source); - String typeParam = param("source_type"); + String typeParam = param("source_content_type"); final XContentType xContentType; if (typeParam != null) { xContentType = parseContentType(Collections.singletonList(typeParam)); diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 46cd40852b4c4..7cb608fdc4b84 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -252,9 +252,9 @@ public void testDispatchRequestLimitsBytes() { } public void testDispatchRequiresContentTypeForRequestsWithContent() { - String content = randomAsciiOfLengthBetween(1, 32); + String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); TestRestRequest request = new TestRestRequest("/", content, null); - AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); + AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); assertFalse(channel.sendResponseCalled.get()); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -271,7 +271,7 @@ public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() { } public void testDispatchWorksWithPlainText() { - String content = randomAsciiOfLengthBetween(1, 32); + String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) .withContent(new BytesArray(content.getBytes(StandardCharsets.UTF_8)), null).withPath("/") .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index 565aad7ffbbac..0324fedd57a98 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -122,7 +122,7 @@ public ClientYamlTestResponse callApi(String apiName, Map params if (supportedMethods.contains("GET") && RandomizedTest.rarely()) { logger.debug("sending the request body as source param with GET method"); queryStringParams.put("source", body); - queryStringParams.put("source_type", ContentType.APPLICATION_JSON.toString()); + queryStringParams.put("source_content_type", ContentType.APPLICATION_JSON.toString()); requestMethod = "GET"; } else { requestMethod = RandomizedTest.randomFrom(supportedMethods); From b5d9e6fac1fa015ba7e0e10d232a57c6a140248f Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 23 Jan 2017 14:55:25 -0500 Subject: [PATCH 09/34] fix line length --- .../org/elasticsearch/percolator/PercolatorQuerySearchIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index cbbd60f958cb6..07fc9aa2fe66f 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -95,7 +95,8 @@ public void testPercolateScriptQuery() throws IOException { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .execute().actionGet(); SearchResponse response = client().prepareSearch("index") - .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON)) + .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), + XContentType.JSON)) .get(); assertHitCount(response, 1); assertSearchHits(response, "1"); From 221f79756217ba099055a0258ef080a5c48278cb Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 23 Jan 2017 15:11:57 -0500 Subject: [PATCH 10/34] fix percolatequerybuilder serialization --- .../org/elasticsearch/percolator/PercolateQueryBuilder.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 561a817fc1d48..2f397a9bdc959 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -210,6 +210,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeOptionalBytesReference(document); + if (document != null && out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + xContentType.writeTo(out); + } } @Override From f43bf3b34a44af2c17e0075ea0f581af20383b45 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 24 Jan 2017 09:35:23 -0500 Subject: [PATCH 11/34] add a BWC compatibility layer that defaults to auto detection fallback --- .../http/HttpTransportSettings.java | 2 + .../elasticsearch/rest/RestController.java | 86 +++++++++++++------ .../org/elasticsearch/rest/RestHandler.java | 5 ++ .../org/elasticsearch/rest/RestRequest.java | 66 ++++++-------- .../action/search/RestClearScrollAction.java | 7 +- .../action/search/RestSearchScrollAction.java | 7 +- .../rest/RestControllerTests.java | 36 +++++--- .../elasticsearch/rest/RestRequestTests.java | 7 +- .../org/elasticsearch/test/ESTestCase.java | 2 +- 9 files changed, 137 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 60bc3449d0be5..b5e254aa4c2ef 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -68,6 +68,8 @@ public final class HttpTransportSettings { Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope); public static final Setting SETTING_HTTP_DETAILED_ERRORS_ENABLED = Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope); + public static final Setting SETTING_HTTP_CONTENT_TYPE_REQUIRED = + Setting.boolSetting("http.content_type.required", false, Property.NodeScope); public static final Setting SETTING_HTTP_MAX_CONTENT_LENGTH = Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope); public static final Setting SETTING_HTTP_MAX_CHUNK_SIZE = diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index ce43f6c9362d5..73693c095de0e 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -43,6 +43,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.common.xcontent.XContentType; @@ -69,6 +71,10 @@ public class RestController extends AbstractComponent { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set headersToCopy; + private final boolean isContentTypeRequired; + + private final DeprecationLogger deprecationLogger; + public RestController(Settings settings, Set headersToCopy, UnaryOperator handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService) { super(settings); @@ -79,10 +85,10 @@ public RestController(Settings settings, Set headersToCopy, UnaryOperato this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; + this.isContentTypeRequired = HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.get(settings); + this.deprecationLogger = new DeprecationLogger(logger); } - - /** * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. * @@ -168,7 +174,8 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont RestChannel responseChannel = channel; try { final int contentLength = request.hasContent() ? request.content().length() : 0; - if (checkForContentType(request, responseChannel, contentLength)) { + final RestHandler handler = getHandler(request); + if (checkForContentType(request, responseChannel, contentLength, handler)) { if (canTripCircuitBreaker(request)) { inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); } else { @@ -176,7 +183,7 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont } // iff we could reserve bytes for the request we need to send the response also over this channel responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); - dispatchRequest(request, responseChannel, client, threadContext); + dispatchRequest(request, responseChannel, client, threadContext, handler); } } catch (Exception e) { try { @@ -189,7 +196,8 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont } } - void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception { + void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext, + final RestHandler handler) throws Exception { if (!checkRequestParameters(request, channel)) { return; } @@ -201,8 +209,6 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final } } - final RestHandler handler = getHandler(request); - if (handler == null) { if (request.method() == RestRequest.Method.OPTIONS) { // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) @@ -219,32 +225,64 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final } /** - * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present and matches an - * {@link XContentType} or the request is plain text. + * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present, matches an + * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method + * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text */ - boolean checkForContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength) { + boolean checkForContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength, + final RestHandler restHandler) { if (contentLength > 0) { - if (restRequest.getXContentType() == null && restRequest.isPlainText() == false) { - final List contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); - final String errorMessage; - if (contentTypeHeader == null) { - errorMessage = "Content-Type header is missing"; + if (restRequest.getXContentType() == null) { + if (restHandler != null && restHandler.supportsPlainText()) { + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else if (isContentTypeRequired) { + sendContentTypeErrorMessage(restRequest, channel); + return false; } else { - errorMessage = "Content-Type header [" + - Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; - } - - try { - channel.sendErrorResponse(NOT_ACCEPTABLE, errorMessage); - } catch (IOException e) { - logger.warn("Failed to send response", e); + deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using" + + "the [Content-Type] header."); + XContentType xContentType = XContentFactory.xContentType(restRequest.content()); + if (xContentType == null) { + if (restHandler != null) { + if (restHandler.supportsPlainText()) { + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else { + try { + channel.sendErrorResponse(NOT_ACCEPTABLE, "Could not determine content type from request body."); + } catch (IOException e) { + logger.warn("Failed to send response", e); + } + return false; + } + } + } else { + restRequest.setxContentType(xContentType); + } } - return false; } } return true; } + private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) { + final List contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); + final String errorMessage; + if (contentTypeHeader == null) { + errorMessage = "Content-Type header is missing"; + } else { + errorMessage = "Content-Type header [" + + Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; + } + + try { + channel.sendErrorResponse(NOT_ACCEPTABLE, errorMessage); + } catch (IOException e) { + logger.warn("Failed to send response", e); + } + } + /** * Checks the request parameters against enabled settings for error trace support * @return true if the request does not have any parameters that conflict with system settings diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index 2d33d6e3dda5d..b061e9ce763fe 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -37,4 +37,9 @@ public interface RestHandler { default boolean canTripCircuitBreaker() { return true; } + + @Deprecated + default boolean supportsPlainText() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index 0029a47cb63c4..eced420b580ef 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; @@ -27,6 +28,8 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -51,6 +54,7 @@ public abstract class RestRequest implements ToXContent.Params { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class)); // tchar pattern as defined by RFC7230 section 3.2.6 private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); @@ -59,8 +63,7 @@ public abstract class RestRequest implements ToXContent.Params { private final Map> headers; private final String rawPath; private final Set consumedParams = new HashSet<>(); - private final XContentType xContentType; - private final boolean isPlainText; + private final SetOnce xContentType = new SetOnce<>(); /** * Creates a new RestRequest @@ -82,8 +85,10 @@ public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map contentType = getAllHeaderValues("Content-Type"); - this.xContentType = parseContentType(contentType); - this.isPlainText = xContentType == null && isPlainTextContentType(contentType); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } /** @@ -100,8 +105,10 @@ public RestRequest(NamedXContentRegistry xContentRegistry, Map p this.rawPath = path; this.headers = Collections.unmodifiableMap(headers); final List contentType = getAllHeaderValues("Content-Type"); - this.xContentType = parseContentType(contentType); - this.isPlainText = xContentType == null && isPlainTextContentType(contentType); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } public enum Method { @@ -166,18 +173,19 @@ public final Map> getHeaders() { /** * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request - * ({@link #isPlainText()}) */ @Nullable public final XContentType getXContentType() { - return xContentType; + return xContentType.get(); } /** - * Returns {@code true} iff the request has content and the {@code Content-Type} header is a plain text header + * Sets the {@link XContentType} + * @deprecated this is only used to allow BWC with content-type detection */ - public final boolean isPlainText() { - return isPlainText; + @Deprecated + final void setxContentType(XContentType xContentType) { + this.xContentType.set(xContentType); } @Nullable @@ -330,10 +338,10 @@ public final XContentParser contentParser() throws IOException { BytesReference content = content(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); - } else if (xContentType == null) { + } else if (xContentType.get() == null) { throw new IllegalStateException("Content-Type must be provided"); } - return xContentType.xContent().createParser(xContentRegistry, content); + return xContentType.get().xContent().createParser(xContentRegistry, content); } /** @@ -393,10 +401,10 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { if (hasContent()) { - if (xContentType == null) { + if (xContentType.get() == null) { throw new IllegalStateException("Content-Type must be provided"); } - return new Tuple<>(xContentType, content()); + return new Tuple<>(xContentType.get(), content()); } String source = param("source"); if (source != null) { @@ -406,6 +414,8 @@ public final Tuple contentOrSourceParam() { if (typeParam != null) { xContentType = parseContentType(Collections.singletonList(typeParam)); } else { + DEPRECATION_LOGGER.deprecated("Deprecated use of the [source] parameter without the [source_content_type] parameter. Use " + + "the [source_content_type] parameter to specify the content type of the source such as [application/json]"); xContentType = XContentFactory.xContentType(bytes); } @@ -425,7 +435,7 @@ public final Tuple contentOrSourceParam() { @Deprecated public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer withParser) throws IOException { - if (hasContent() && isPlainText) { + if (hasContent() && xContentType.get() == null) { withParser.accept(null); } @@ -482,28 +492,4 @@ private static XContentType parseContentType(List header) { } return null; } - - /** - * Parses the Content-Type header and returns true if the header matches the plain text media type - */ - private static boolean isPlainTextContentType(List header) { - if (header == null || header.isEmpty()) { - return false; - } else if (header.size() > 1) { - throw new IllegalArgumentException("only one Content-Type header should be provided"); - } - - String rawContentType = header.get(0); - final String[] elements = rawContentType.split("[ \t]*;"); - if (elements.length > 0) { - final String[] splitMediaType = elements[0].split("/"); - if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() - && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { - return splitMediaType[0].equals("text") && (splitMediaType[1].equals("plain") || splitMediaType[1].equals("plain;")); - } else { - throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); - } - } - return false; - } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java index 45a1e650d1c80..47252f5a10217 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java @@ -50,7 +50,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); request.withContentOrSourceParamParserOrNullLenient((xContentParser -> { if (xContentParser == null) { - if (request.hasContent() && request.isPlainText()) { + if (request.hasContent()) { // TODO: why do we accept this plain text value? maybe we can just use the scroll params? BytesReference body = request.content(); String bodyScrollIds = body.utf8ToString(); @@ -70,6 +70,11 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel)); } + @Override + public boolean supportsPlainText() { + return true; + } + private static String[] splitScrollIds(String scrollIds) { if (scrollIds == null) { return Strings.EMPTY_ARRAY; diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index 3065bc6df67c2..2a60fc6317a3c 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -59,7 +59,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.withContentOrSourceParamParserOrNull(xContentParser -> { if (xContentParser == null) { - if (request.hasContent() && request.isPlainText()) { + if (request.hasContent()) { // TODO: why do we accept this plain text value? maybe we can just use the scroll params? BytesReference body = request.getContentOrSourceParamOnly(); if (scrollId == null) { @@ -79,6 +79,11 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } + @Override + public boolean supportsPlainText() { + return true; + } + public static void buildFromContent(XContentParser parser, SearchScrollRequest searchScrollRequest) throws IOException { if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Malformed content, must start with an object"); diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 7cb608fdc4b84..26be37c1ea1a6 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.http.HttpInfo; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.HttpStats; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -93,19 +94,17 @@ public void testApplyRelevantHeaders() throws Exception { final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); Set headers = new HashSet<>(Arrays.asList("header.1", "header.2")); final RestController restController = new RestController(Settings.EMPTY, headers, null, null, circuitBreakerService); - restController.registerHandler(RestRequest.Method.GET, "/", - (RestRequest request, RestChannel channel, NodeClient client) -> { - assertEquals("true", threadContext.getHeader("header.1")); - assertEquals("true", threadContext.getHeader("header.2")); - assertNull(threadContext.getHeader("header.3")); - }); threadContext.putHeader("header.3", "true"); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("true")); restHeaders.put("header.2", Collections.singletonList("true")); restHeaders.put("header.3", Collections.singletonList("false")); restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null, - threadContext); + threadContext, (RestRequest request, RestChannel channel, NodeClient client) -> { + assertEquals("true", threadContext.getHeader("header.1")); + assertEquals("true", threadContext.getHeader("header.2")); + assertNull(threadContext.getHeader("header.3")); + }); assertNull(threadContext.getHeader("header.1")); assertNull(threadContext.getHeader("header.2")); assertEquals("true", threadContext.getHeader("header.3")); @@ -174,9 +173,8 @@ public void testRestHandlerWrapper() throws Exception { }; final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper, null, circuitBreakerService); - restController.registerHandler(RestRequest.Method.GET, "/", handler); final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext); + restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext, handler); assertTrue(wrapperCalled.get()); assertFalse(handlerCalled.get()); } @@ -255,6 +253,12 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() { String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); TestRestRequest request = new TestRestRequest("/", content, null); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); + restController = new RestController( + Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(), + Collections.emptySet(), null, null, circuitBreakerService); + restController.registerHandler(RestRequest.Method.GET, "/", + (r, c, client) -> c.sendResponse( + new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY))); assertFalse(channel.sendResponseCalled.get()); restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY)); @@ -273,13 +277,25 @@ public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() { public void testDispatchWorksWithPlainText() { String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) - .withContent(new BytesArray(content.getBytes(StandardCharsets.UTF_8)), null).withPath("/") + .withContent(new BytesArray(content.getBytes(StandardCharsets.UTF_8)), null).withPath("/foo") .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build(); AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + } + + @Override + public boolean supportsPlainText() { + return true; + } + }); assertFalse(channel.sendResponseCalled.get()); restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); assertTrue(channel.sendResponseCalled.get()); + assertWarnings("Plain text request bodies are deprecated. Use request parameters or body in a supported format."); } private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java index d96c3212e3222..bddcd39160a89 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -64,6 +64,7 @@ public void testContentOrSourceParam() throws IOException { new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam().v2()); assertEquals(new BytesArray("{\"foo\": \"stuff\"}"), new ContentRestRequest("", singletonMap("source", "{\"foo\": \"stuff\"}")).contentOrSourceParam().v2()); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); } public void testHasContentOrSourceParam() throws IOException { @@ -80,6 +81,7 @@ public void testContentOrSourceParamParser() throws IOException { assertEquals(emptyMap(), new ContentRestRequest("{}", emptyMap()).contentOrSourceParamParser().map()); assertEquals(emptyMap(), new ContentRestRequest("{}", singletonMap("source", "stuff2")).contentOrSourceParamParser().map()); assertEquals(emptyMap(), new ContentRestRequest("", singletonMap("source", "{}")).contentOrSourceParamParser().map()); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); } public void testWithContentOrSourceParamParserOrNull() throws IOException { @@ -89,6 +91,7 @@ public void testWithContentOrSourceParamParserOrNull() throws IOException { assertEquals(emptyMap(), parser.map())); new ContentRestRequest("", singletonMap("source", "{}")).withContentOrSourceParamParserOrNull(parser -> assertEquals(emptyMap(), parser.map())); + assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter."); } public void testContentTypeParsing() { @@ -97,13 +100,11 @@ public void testContentTypeParsing() { map.put("Content-Type", Collections.singletonList(xContentType.mediaType())); ContentRestRequest restRequest = new ContentRestRequest("", Collections.emptyMap(), map); assertEquals(xContentType, restRequest.getXContentType()); - assertFalse(restRequest.isPlainText()); map = new HashMap<>(); map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters())); restRequest = new ContentRestRequest("", Collections.emptyMap(), map); assertEquals(xContentType, restRequest.getXContentType()); - assertFalse(restRequest.isPlainText()); } } @@ -112,7 +113,6 @@ public void testPlainTextSupport() { Collections.singletonMap("Content-Type", Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8")))); assertNull(restRequest.getXContentType()); - assertTrue(restRequest.isPlainText()); } public void testMalformedContentTypeHeader() { @@ -125,7 +125,6 @@ public void testMalformedContentTypeHeader() { public void testNoContentTypeHeader() { ContentRestRequest contentRestRequest = new ContentRestRequest("", Collections.emptyMap(), Collections.emptyMap()); assertNull(contentRestRequest.getXContentType()); - assertFalse(contentRestRequest.isPlainText()); } public void testMultipleContentTypeHeaders() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index d4c24ccb617f3..0349ddd0822d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -290,7 +290,7 @@ public final void after() throws Exception { } private void ensureNoWarnings() throws IOException { - //Check that there are no unaccounted warning headers. These should be checked with {@link #checkWarningHeaders(String...)} in the + //Check that there are no unaccounted warning headers. These should be checked with {@link #assertWarnings(String...)} in the //appropriate test try { final List warnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER); From c6be5401816b824afbe2f764202dda52bde691c7 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 24 Jan 2017 14:50:38 -0500 Subject: [PATCH 12/34] serialization tests and updates --- .../storedscripts/PutStoredScriptRequest.java | 9 +-- .../indices/create/CreateIndexRequest.java | 3 +- .../mapping/get/GetFieldMappingsResponse.java | 10 +++ .../PutStoredScriptRequestTests.java | 53 ++++++++----- .../create/CreateIndexRequestTests.java | 75 ++++++++++++++++++ .../get/GetFieldMappingsResponseTests.java | 77 +++++++++++++++++++ .../mapping/put/PutMappingRequestTests.java | 39 ++++++---- .../put/PutIndexTemplateRequestTests.java | 33 +++++--- .../action/index/IndexRequestTests.java | 31 ++++---- .../ingest/PutPipelineRequestTests.java | 31 ++++---- .../ingest/SimulatePipelineRequestTests.java | 31 ++++---- .../termvectors/TermVectorsUnitTests.java | 36 +++++++++ .../query/MoreLikeThisQueryBuilderTests.java | 24 ++++++ .../ingest/PipelineConfigurationTests.java | 31 +++++--- .../percolator/PercolateQueryBuilder.java | 5 ++ .../PercolateQueryBuilderTests.java | 24 ++++++ 16 files changed, 406 insertions(+), 106 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 9b7d0feb9e508..b90af5441aad5 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -127,8 +127,10 @@ public void readFrom(StreamInput in) throws IOException { scriptLang = in.readString(); id = in.readOptionalString(); script = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED) && in.readBoolean()) { + if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(script); } } @@ -139,12 +141,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBytesReference(script); if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - if (xContentType == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); xContentType.writeTo(out); - } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 49fd0044c1994..c8a42abf97ac2 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -551,9 +551,10 @@ public void readFrom(StreamInput in) throws IOException { final XContentType xContentType = XContentType.readFrom(in); mappings.put(type, new Tuple<>(xContentType, bytesReference)); } else { + final String type = in.readString(); final BytesReference bytesReference = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); final XContentType xContentType = XContentFactory.xContentType(bytesReference); - mappings.put(in.readString(), new Tuple<>(xContentType, bytesReference)); + mappings.put(type, new Tuple<>(xContentType, bytesReference)); } } int customSize = in.readVInt(); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index ede90ba5f3528..89f42831cc529 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -120,6 +120,16 @@ public boolean isNull() { return NULL.fullName().equals(fullName) && NULL.source.length() == source.length(); } + //pkg-private for testing + XContentType getXContentType() { + return xContentType; + } + + //pkg-private for testing + BytesReference getSource() { + return source; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("full_name", fullName); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index 7716888a92482..304a89cfa1e5b 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Base64; public class PutStoredScriptRequestTests extends ESTestCase { @@ -35,26 +36,38 @@ public void testSerialization() throws IOException { storedScriptRequest.script(new BytesArray("{}"), XContentType.JSON); assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); - BytesStreamOutput output = new BytesStreamOutput(); - storedScriptRequest.writeTo(output); + try (BytesStreamOutput output = new BytesStreamOutput()) { + storedScriptRequest.writeTo(output); - StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); - PutStoredScriptRequest serialized = new PutStoredScriptRequest(); - serialized.readFrom(in); - assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); - assertEquals(storedScriptRequest.scriptLang(), serialized.scriptLang()); - assertEquals(storedScriptRequest.id(), serialized.id()); - - // send to an old version and then read it - output = new BytesStreamOutput(); - output.setVersion(Version.V_5_0_0); - storedScriptRequest.writeTo(output); - in = StreamInput.wrap(output.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); - serialized = new PutStoredScriptRequest(); - serialized.readFrom(in); - assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); - assertEquals(storedScriptRequest.scriptLang(), serialized.scriptLang()); - assertEquals(storedScriptRequest.id(), serialized.id()); + try (StreamInput in = output.bytes().streamInput()) { + PutStoredScriptRequest serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals(storedScriptRequest.scriptLang(), serialized.scriptLang()); + assertEquals(storedScriptRequest.id(), serialized.id()); + } + } + } + + public void testSerializationBwc() throws IOException { + final byte[] rawStreamBytes = Base64.getDecoder().decode("ADwDCG11c3RhY2hlAQZzY3JpcHQCe30A"); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(rawStreamBytes)) { + in.setVersion(version); + PutStoredScriptRequest serialized = new PutStoredScriptRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals("mustache", serialized.scriptLang()); + assertEquals("script", serialized.id()); + assertEquals(new BytesArray("{}"), serialized.script()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + serialized.writeTo(out); + out.flush(); + assertArrayEquals(rawStreamBytes, out.bytes().toBytesRef().bytes); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java new file mode 100644 index 0000000000000..18ac79b09efa9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Base64; + +public class CreateIndexRequestTests extends ESTestCase { + + public void testSerialization() throws IOException { + CreateIndexRequest request = new CreateIndexRequest("foo"); + BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); + request.mapping("my_type", bytesReference, XContentType.JSON); + + assertEquals(XContentType.JSON, request.mappings().get("my_type").v1()); + try (BytesStreamOutput output = new BytesStreamOutput()) { + request.writeTo(output); + + try (StreamInput in = output.bytes().streamInput()) { + CreateIndexRequest serialized = new CreateIndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.mappings().get("my_type").v1()); + assertEquals(request.index(), serialized.index()); + assertEquals(bytesReference, serialized.mappings().get("my_type").v2()); + } + } + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAANmb28APAMBB215X3R5cGULeyJ0eXBlIjp7fX0AAAD////+AA=="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + CreateIndexRequest serialized = new CreateIndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.mappings().get("my_type").v1()); + assertEquals("foo", serialized.index()); + BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); + assertEquals(bytesReference, serialized.mappings().get("my_type").v2()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + serialized.writeTo(out); + out.flush(); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java new file mode 100644 index 0000000000000..a9dde649ff673 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.mapping.get; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class GetFieldMappingsResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + Map>> mappings = new HashMap<>(); + FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}"), XContentType.JSON); + mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData))); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + GetFieldMappingsResponse serialized = new GetFieldMappingsResponse(); + try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) { + serialized.readFrom(in); + FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field"); + assertNotNull(metaData); + assertEquals(XContentType.JSON, metaData.getXContentType()); + assertEquals(new BytesArray("{}"), metaData.getSource()); + } + } + } + + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEFZmllbGQIbXkgZmllbGQCe30="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(); + response.readFrom(in); + FieldMappingMetaData metaData = response.fieldMappings("index", "type", "field"); + assertNotNull(metaData); + assertEquals(XContentType.JSON, metaData.getXContentType()); + assertEquals(new BytesArray("{}"), metaData.getSource()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + response.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index a9fd6c5450039..284dc19eca335 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Base64; public class PutMappingRequestTests extends ESTestCase { @@ -73,34 +74,40 @@ public void testBuildFromSimplifiedDef() { assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage()); } - public void testPutMappingRequestFromOldVersion() throws IOException { - // this is hacky but here goes + public void testPutMappingRequestSerialization() throws IOException { PutMappingRequest request = new PutMappingRequest("foo"); String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); - request.source(mapping, XContentType.JSON); + request.source(mapping, XContentType.YAML); assertEquals(mapping, request.source().utf8ToString()); - // output version doesn't matter BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - bytesStreamOutput.setVersion(Version.V_5_0_0); request.writeTo(bytesStreamOutput); - StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); PutMappingRequest serialized = new PutMappingRequest(); serialized.readFrom(in); BytesReference source = serialized.source(); assertEquals(mapping, source.utf8ToString()); + assertEquals(XContentType.YAML, request.getXContentType()); + } - // reading from a fixed version does no translation - bytesStreamOutput = new BytesStreamOutput(); - request.writeTo(bytesStreamOutput); - in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes); - assertEquals(Version.CURRENT, in.getVersion()); - serialized = new PutMappingRequest(); - serialized.readFrom(in); - assertEquals(mapping, serialized.source().utf8ToString()); - assertEquals(XContentType.JSON, serialized.getXContentType()); + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAQNmb28MAA8tLS0KZm9vOiAiYmFyIgoAPAMAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutMappingRequest request = new PutMappingRequest(); + request.readFrom(in); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + assertEquals(mapping, request.source().utf8ToString()); + assertEquals(XContentType.YAML, request.getXContentType()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index b8dc16274ec04..e2cc3f74cc8ea 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -73,27 +73,38 @@ public void testPutIndexTemplateRequestSerializationXContent() throws IOExceptio PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); BytesReference mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); request.patterns(Collections.singletonList("foo")); - // THIS IS NOT A BUG! Intentionally specifying the wrong type so we serialize it - request.mapping("bar", mapping, XContentType.JSON); + request.mapping("bar", mapping, XContentType.YAML); assertEquals(mapping, request.mappings().get("bar").v2()); BytesStreamOutput out = new BytesStreamOutput(); - out.setVersion(Version.V_5_0_0); request.writeTo(out); StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); serialized.readFrom(in); assertEquals(mapping, serialized.mappings().get("bar").v2()); assertEquals(XContentType.YAML, serialized.mappings().get("bar").v1()); + } - out = new BytesStreamOutput(); - request.writeTo(out); - in = StreamInput.wrap(out.bytes().toBytesRef().bytes); - serialized = new PutIndexTemplateRequest(); - serialized.readFrom(in); - assertEquals(mapping, serialized.mappings().get("bar").v2()); - assertEquals(XContentType.JSON, serialized.mappings().get("bar").v1()); + public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDAANmb28IdGVtcGxhdGUAAAAAAAABA2Jhcg8tLS0KZm9vOiAiYmFyIgoAAAAAAAAAAAAAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.readFrom(in); + assertEquals(XContentType.YAML, request.mappings().get("bar").v1()); + BytesReference mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); + assertEquals(mapping, request.mappings().get("bar").v2()); + assertEquals("foo", request.name()); + assertEquals("template", request.patterns().get(0)); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index bd2fc5bc335e8..39463d9f19866 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; @@ -34,6 +35,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.HashSet; import java.util.Set; @@ -167,21 +169,20 @@ public void testIndexRequestXContentSerialization() throws IOException { IndexRequest serialized = new IndexRequest(); serialized.readFrom(in); assertEquals(XContentType.JSON, serialized.getContentType()); + assertEquals(new BytesArray("{}"), serialized.source()); + } - // test with an incorrect content type and send it to an old version then see that we get the right content type when reading - indexRequest = new IndexRequest("foo", "bar", "1"); - indexRequest.source("{}", XContentType.YAML); - assertEquals(XContentType.YAML, indexRequest.getContentType()); - - out = new BytesStreamOutput(); - out.setVersion(Version.V_5_0_0); - indexRequest.writeTo(out); - in = StreamInput.wrap(out.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); - - serialized = new IndexRequest(); - serialized.readFrom(in); - assertEquals(XContentType.JSON, serialized.getContentType()); - assertEquals("{}", serialized.source().utf8ToString()); + public void testIndexRequestXContentSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAD////+AgQDZm9vAAAAAQNiYXIBATEAAAAAAnt9AP/////////9AAAA//////////8AAAAAAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertEquals(XContentType.JSON, serialized.getContentType()); + assertEquals("{}", serialized.source().utf8ToString()); + // don't test writing to earlier versions since output differs due to no timestamp + } } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java index 33a1be5f337f4..01aed87947acf 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Base64; public class PutPipelineRequestTests extends ESTestCase { @@ -43,20 +44,24 @@ public void testSerializationWithXContent() throws IOException { serialized.readFrom(in); assertEquals(XContentType.JSON, serialized.getXContentType()); assertEquals("{}", serialized.getSource().utf8ToString()); + } - // send to old and read from old with a bad content type and see that we find the correct one - request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); - assertEquals(XContentType.YAML, request.getXContentType()); - - output = new BytesStreamOutput(); - output.setVersion(Version.V_5_0_0); - request.writeTo(output); - in = StreamInput.wrap(output.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ADwDATECe30="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PutPipelineRequest request = new PutPipelineRequest(); + request.readFrom(in); + assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals("{}", request.getSource().utf8ToString()); - serialized = new PutPipelineRequest(); - serialized.readFrom(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); - assertEquals("{}", serialized.getSource().utf8ToString()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java index bdc7efa92c049..86dc56cdd04c0 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.hamcrest.CoreMatchers.equalTo; @@ -68,20 +69,24 @@ public void testSerializationWithXContent() throws IOException { serialized.readFrom(in); assertEquals(XContentType.JSON, serialized.getXContentType()); assertEquals("{}", serialized.getSource().utf8ToString()); + } - // send to old and read from old with a bad content type and see that we find the correct one - request = new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); - assertEquals(XContentType.YAML, request.getXContentType()); - - output = new BytesStreamOutput(); - output.setVersion(Version.V_5_0_0); - request.writeTo(output); - in = StreamInput.wrap(output.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); + public void testSerializationWithXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAAAAnt9AAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + SimulatePipelineRequest request = new SimulatePipelineRequest(); + request.readFrom(in); + assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals("{}", request.getSource().utf8ToString()); - serialized = new SimulatePipelineRequest(); - serialized.readFrom(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); - assertEquals("{}", serialized.getSource().utf8ToString()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java index c255ef6fc884f..e034cff3f1de0 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java @@ -36,12 +36,16 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -56,6 +60,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.EnumSet; import java.util.HashSet; import java.util.Set; @@ -236,6 +241,7 @@ public void testStreamRequest() throws IOException { request.parent(parent); String pref = random().nextBoolean() ? "somePreference" : null; request.preference(pref); + request.doc(new BytesArray("{}"), randomBoolean(), XContentType.JSON); // write ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); @@ -255,7 +261,37 @@ public void testStreamRequest() throws IOException { assertThat(request.termStatistics(), equalTo(req2.termStatistics())); assertThat(request.preference(), equalTo(pref)); assertThat(request.routing(), equalTo(null)); + assertEquals(new BytesArray("{}"), request.doc()); + assertEquals(XContentType.JSON, request.xContentType()); + } + } + public void testStreamRequestWithXContentBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AAABBWluZGV4BHR5cGUCaWQBAnt9AAABDnNvbWVQcmVmZXJlbmNlFgAAAAEA//////////0AAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + TermVectorsRequest request = new TermVectorsRequest(); + request.readFrom(in); + assertEquals("index", request.index()); + assertEquals("type", request.type()); + assertEquals("id", request.id()); + assertTrue(request.offsets()); + assertFalse(request.fieldStatistics()); + assertTrue(request.payloads()); + assertFalse(request.positions()); + assertTrue(request.termStatistics()); + assertNull(request.parent()); + assertEquals("somePreference", request.preference()); + assertEquals("{}", request.doc().utf8ToString()); + assertEquals(XContentType.JSON, request.xContentType()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } } } diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java index 8daf45ed0dd01..ee1265eca656f 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -26,17 +26,20 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; @@ -46,6 +49,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -301,6 +305,26 @@ public void testItemFromXContent() throws IOException { assertEquals(expectedItem, newItem); } + public void testItemSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEODXsiZm9vIjoiYmFyIn0A/wD//////////QAAAAAAAAAA"); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + Item item = new Item(in); + assertEquals(XContentType.JSON, item.xContentType()); + assertEquals("{\"foo\":\"bar\"}", item.doc().utf8ToString()); + assertEquals("index", item.index()); + assertEquals("type", item.type()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + item.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } + @Override protected boolean isCachable(MoreLikeThisQueryBuilder queryBuilder) { return queryBuilder.likeItems().length == 0; // items are always fetched diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java index 6aaac41b8beab..f41d01b32c89d 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -35,30 +35,39 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Base64; public class PipelineConfigurationTests extends ESTestCase { public void testSerialization() throws IOException { PipelineConfiguration configuration = new PipelineConfiguration("1", - new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.YAML); - assertEquals(XContentType.YAML, configuration.getXContentType()); + new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + assertEquals(XContentType.JSON, configuration.getXContentType()); BytesStreamOutput out = new BytesStreamOutput(); configuration.writeTo(out); StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); PipelineConfiguration serialized = PipelineConfiguration.readFrom(in); - assertEquals(XContentType.YAML, serialized.getXContentType()); + assertEquals(XContentType.JSON, serialized.getXContentType()); assertEquals("{}", serialized.getConfig().utf8ToString()); + } - out = new BytesStreamOutput(); - out.setVersion(Version.V_5_0_0); - configuration.writeTo(out); - in = StreamInput.wrap(out.bytes().toBytesRef().bytes); - in.setVersion(Version.V_5_0_0); + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("ATECe30AAAA="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PipelineConfiguration configuration = PipelineConfiguration.readFrom(in); + assertEquals(XContentType.JSON, configuration.getXContentType()); + assertEquals("{}", configuration.getConfig().utf8ToString()); - serialized = PipelineConfiguration.readFrom(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); - assertEquals("{}", serialized.getConfig().utf8ToString()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + configuration.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } } public void testParser() throws IOException { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 2f397a9bdc959..009616281e663 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -461,6 +461,11 @@ public BytesReference getDocument() { return document; } + //pkg-private for testing + XContentType getXContentType() { + return xContentType; + } + static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) { RAMDirectory ramDirectory = new RAMDirectory(); try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index e8f07830a09b4..6f1e80d80b8f7 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -27,12 +27,15 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -49,6 +52,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -243,6 +247,26 @@ public void testCreateMultiDocumentSearcher() throws Exception { assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT)); } + public void testSerializationBwc() throws IOException { + final byte[] data = Base64.getDecoder().decode("P4AAAAAFZmllbGQEdHlwZQAAAAAAAA57ImZvbyI6ImJhciJ9AAAAAA=="); + final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, + Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); + try (StreamInput in = StreamInput.wrap(data)) { + in.setVersion(version); + PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(in); + assertEquals("type", queryBuilder.getDocumentType()); + assertEquals("field", queryBuilder.getField()); + assertEquals("{\"foo\":\"bar\"}", queryBuilder.getDocument().utf8ToString()); + assertEquals(XContentType.JSON, queryBuilder.getXContentType()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + queryBuilder.writeTo(out); + assertArrayEquals(data, out.bytes().toBytesRef().bytes); + } + } + } + private static BytesReference randomSource() { try { XContentBuilder xContent = XContentFactory.jsonBuilder(); From 0be9af3ad0262a9af7cee3c9121b7d1969fb4f33 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 12:50:59 -0500 Subject: [PATCH 13/34] review round 2 updates --- .../put/PutRepositoryRequest.java | 2 +- .../restore/RestoreSnapshotRequest.java | 4 +- .../storedscripts/PutStoredScriptRequest.java | 6 +- .../CreateIndexClusterStateUpdateRequest.java | 9 +-- .../indices/create/CreateIndexRequest.java | 65 +++++++-------- .../mapping/get/GetFieldMappingsResponse.java | 31 ++------ .../TransportGetFieldMappingsIndexAction.java | 2 +- .../PutMappingClusterStateUpdateRequest.java | 15 +--- .../mapping/put/PutMappingRequest.java | 54 ++++++------- .../put/TransportPutMappingAction.java | 2 +- .../template/put/PutIndexTemplateRequest.java | 69 ++++++++-------- .../put/PutIndexTemplateRequestBuilder.java | 4 +- .../action/index/IndexRequest.java | 40 ++++------ .../action/ingest/PutPipelineRequest.java | 2 +- .../ingest/SimulatePipelineRequest.java | 2 +- .../metadata/IndexTemplateMetaData.java | 6 +- .../metadata/MetaDataCreateIndexService.java | 17 ++-- .../MetaDataIndexTemplateService.java | 18 ++--- .../metadata/MetaDataMappingService.java | 4 +- .../common/compress/CompressedXContent.java | 9 --- .../common/settings/Settings.java | 1 - .../loader/SettingsLoaderFactory.java | 4 +- .../common/xcontent/XContentGenerator.java | 24 ++++++ .../common/xcontent/XContentType.java | 34 ++------ .../index/mapper/DocumentMapper.java | 6 +- .../index/mapper/DocumentMapperParser.java | 15 +--- .../index/mapper/MapperService.java | 19 ++--- .../index/mapper/SourceFieldMapper.java | 3 +- .../index/mapper/SourceToParse.java | 4 +- .../ingest/PipelineConfiguration.java | 10 +-- .../rest/AbstractRestChannel.java | 34 ++++---- .../elasticsearch/rest/BytesRestResponse.java | 7 ++ .../org/elasticsearch/rest/RestChannel.java | 6 -- .../elasticsearch/rest/RestController.java | 79 ++++++++----------- .../org/elasticsearch/rest/RestRequest.java | 8 +- .../admin/indices/RestPutMappingAction.java | 2 +- .../elasticsearch/script/ScriptMetaData.java | 4 +- .../create/CreateIndexRequestTests.java | 7 +- .../get/GetFieldMappingsResponseTests.java | 27 +------ .../mapping/put/PutMappingRequestTests.java | 19 ++--- .../MetaDataIndexTemplateServiceTests.java | 12 ++- .../put/PutIndexTemplateRequestTests.java | 22 ++---- .../metadata/MetaDataMappingServiceTests.java | 9 +-- .../index/engine/InternalEngineTests.java | 4 +- .../index/mapper/MapperServiceTests.java | 5 +- .../mapper/MultiFieldCopyToMapperTests.java | 2 +- .../MultiFieldIncludeInAllMapperTests.java | 2 +- .../index/mapper/SourceFieldMapperTests.java | 8 +- .../template/SimpleIndexTemplateIT.java | 5 +- .../percolator/PercolateQueryBuilder.java | 20 ++--- .../remote/RemoteScrollableHitSource.java | 2 +- .../test/rest/FakeRestChannel.java | 4 +- 52 files changed, 310 insertions(+), 459 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index f37a5d0130c75..2e78b51084df8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -144,7 +144,7 @@ public PutRepositoryRequest settings(Settings.Builder settings) { * * @param source repository settings in json, yaml or properties format * @return this request - * @deprecated use {@link #settings(String, XContentType)} to avoid content auto-detection + * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection */ @Deprecated public PutRepositoryRequest settings(String source) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 67ffa9a716893..9d8ed49aaa09f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -313,7 +313,7 @@ public RestoreSnapshotRequest settings(Settings.Builder settings) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * @@ -328,7 +328,7 @@ public RestoreSnapshotRequest settings(String source) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index b90af5441aad5..2603bca73c482 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -107,9 +107,7 @@ public XContentType xContentType() { */ @Deprecated public PutStoredScriptRequest script(BytesReference source) { - this.script = source; - this.xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); - return this; + return script(source, XContentFactory.xContentType(source)); } /** @@ -141,7 +139,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBytesReference(script); if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - xContentType.writeTo(out); + xContentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 9a73e99acf0f9..a2290a5e2556e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -24,10 +24,7 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.transport.TransportMessage; @@ -52,7 +49,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private Settings settings = Settings.Builder.EMPTY_SETTINGS; - private final Map> mappings = new HashMap<>(); + private final Map mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -77,7 +74,7 @@ public CreateIndexClusterStateUpdateRequest settings(Settings settings) { return this; } - public CreateIndexClusterStateUpdateRequest mappings(Map> mappings) { + public CreateIndexClusterStateUpdateRequest mappings(Map mappings) { this.mappings.putAll(mappings); return this; } @@ -132,7 +129,7 @@ public Settings settings() { return settings; } - public Map> mappings() { + public Map mappings() { return mappings; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index c8a42abf97ac2..b0d0ffc5c8663 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -45,6 +44,7 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; @@ -74,7 +74,7 @@ public class CreateIndexRequest extends AcknowledgedRequest private Settings settings = EMPTY_SETTINGS; - private final Map> mappings = new HashMap<>(); + private final Map mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -225,12 +225,7 @@ public CreateIndexRequest settings(Map source) { */ @Deprecated public CreateIndexRequest mapping(String type, String source) { - if (mappings.containsKey(type)) { - throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); - } - XContentType xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); - mappings.put(type, new Tuple<>(xContentType, new BytesArray(source.getBytes(StandardCharsets.UTF_8)))); - return this; + return mapping(type, new BytesArray(source), XContentFactory.xContentType(source)); } /** @@ -245,7 +240,7 @@ public CreateIndexRequest mapping(String type, BytesReference source, XContentTy throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } Objects.requireNonNull(xContentType); - mappings.put(type, new Tuple<>(xContentType, source)); + mappings.put(type, convertToJsonIfNecessary(source, xContentType)); return this; } @@ -264,11 +259,7 @@ public CreateIndexRequest cause(String cause) { * @param source The mapping source */ public CreateIndexRequest mapping(String type, XContentBuilder source) { - if (mappings.containsKey(type)) { - throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); - } - mappings.put(type, new Tuple<>(source.contentType(), source.bytes())); - return this; + return mapping(type, source.bytes(), source.contentType()); } /** @@ -289,7 +280,7 @@ public CreateIndexRequest mapping(String type, Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -472,7 +463,7 @@ public CreateIndexRequest source(Map source) { return this; } - public Map> mappings() { + public Map mappings() { return this.mappings; } @@ -545,17 +536,12 @@ public void readFrom(StreamInput in) throws IOException { readTimeout(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - final String type = in.readString(); - final BytesReference bytesReference = in.readBytesReference(); - final XContentType xContentType = XContentType.readFrom(in); - mappings.put(type, new Tuple<>(xContentType, bytesReference)); - } else { - final String type = in.readString(); - final BytesReference bytesReference = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); - final XContentType xContentType = XContentFactory.xContentType(bytesReference); - mappings.put(type, new Tuple<>(xContentType, bytesReference)); + final String type = in.readString(); + String source = in.readString(); + if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); } + mappings.put(type, source); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { @@ -579,17 +565,9 @@ public void writeTo(StreamOutput out) throws IOException { writeSettingsToStream(settings, out); writeTimeout(out); out.writeVInt(mappings.size()); - for (Map.Entry> entry : mappings.entrySet()) { + for (Map.Entry entry : mappings.entrySet()) { out.writeString(entry.getKey()); - final Tuple value = entry.getValue(); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - out.writeBytesReference(value.v2()); - value.v1().writeTo(out); - } else if (value.v1().hasStringRepresentation()) { - out.writeString(value.v2().utf8ToString()); - } else { - throw new IllegalStateException("cannot send [" + value.v1() + "] to an older node"); - } + out.writeString(entry.getValue()); } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { @@ -603,4 +581,19 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(updateAllTypes); waitForActiveShards.writeTo(out); } + + private String convertToJsonIfNecessary(String source, XContentType xContentType) { + return convertToJsonIfNecessary(new BytesArray(source), xContentType); + } + + private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return source.utf8ToString(); + } + try { + return XContentHelper.convertToJson(source, false, xContentType); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to JSON", e); + } + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 89f42831cc529..3f4ddaf08db2c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.indices.mapping.get; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -27,7 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; @@ -95,16 +93,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static class FieldMappingMetaData implements ToXContent { - public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY, XContentType.JSON); + public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY); private String fullName; private BytesReference source; - private XContentType xContentType; - public FieldMappingMetaData(String fullName, BytesReference source, XContentType xContentType) { + public FieldMappingMetaData(String fullName, BytesReference source) { this.fullName = fullName; this.source = source; - this.xContentType = xContentType; } public String fullName() { @@ -113,18 +109,13 @@ public String fullName() { /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */ public Map sourceAsMap() { - return XContentHelper.convertToMap(source, true, xContentType).v2(); + return XContentHelper.convertToMap(source, true, XContentType.JSON).v2(); } public boolean isNull() { return NULL.fullName().equals(fullName) && NULL.source.length() == source.length(); } - //pkg-private for testing - XContentType getXContentType() { - return xContentType; - } - //pkg-private for testing BytesReference getSource() { return source; @@ -136,7 +127,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (params.paramAsBoolean("pretty", false)) { builder.field("mapping", sourceAsMap()); } else { - builder.rawField("mapping", source, xContentType); + builder.rawField("mapping", source, XContentType.JSON); } return builder; } @@ -156,16 +147,7 @@ public void readFrom(StreamInput in) throws IOException { int fieldSize = in.readVInt(); Map fieldMapBuilder = new HashMap<>(fieldSize); for (int k = 0; k < fieldSize; k++) { - final String key = in.readString(); - final String field = in.readString(); - final BytesReference bytesReference = in.readBytesReference(); - final XContentType xContentType; - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - xContentType = XContentType.readFrom(in); - } else { - xContentType = XContentFactory.xContentType(bytesReference); - } - fieldMapBuilder.put(key, new FieldMappingMetaData(field, bytesReference, xContentType)); + fieldMapBuilder.put(in.readString(), new FieldMappingMetaData(in.readString(), in.readBytesReference())); } typeMapBuilder.put(type, unmodifiableMap(fieldMapBuilder)); } @@ -189,9 +171,6 @@ public void writeTo(StreamOutput out) throws IOException { FieldMappingMetaData fieldMapping = fieldEntry.getValue(); out.writeString(fieldMapping.fullName()); out.writeBytesReference(fieldMapping.source); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - fieldMapping.xContentType.writeTo(out); - } } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index a26e864ba53e7..ec84ce4e34779 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -214,7 +214,7 @@ private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuilder im private String type; - private BytesReference source; - private XContentType xContentType; + private String source; private boolean updateAllTypes = false; private Index concreteIndex; @@ -96,7 +96,7 @@ public ActionRequestValidationException validate() { } if (source == null) { validationException = addValidationError("mapping source is missing", validationException); - } else if (source == null || source.length() == 0) { + } else if (source.isEmpty()) { validationException = addValidationError("mapping source is empty", validationException); } if (concreteIndex != null && (indices != null && indices.length > 0)) { @@ -167,17 +167,10 @@ public PutMappingRequest type(String type) { /** * The mapping source definition. */ - public BytesReference source() { + public String source() { return source; } - /** - * The type of content held in the source - */ - public XContentType getXContentType() { - return xContentType; - } - /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). @@ -290,8 +283,8 @@ public PutMappingRequest source(String mappingSource) { * The mapping source definition. */ public PutMappingRequest source(String mappingSource, XContentType xContentType) { - this.source = new BytesArray(mappingSource.getBytes(StandardCharsets.UTF_8)); - this.xContentType = Objects.requireNonNull(xContentType); + Objects.requireNonNull(xContentType); + this.source = convertToJsonIfNecessary(mappingSource, xContentType); return this; } @@ -312,12 +305,9 @@ public void readFrom(StreamInput in) throws IOException { indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - source = in.readBytesReference(); - xContentType = XContentType.readFrom(in); - } else { - source = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); - xContentType = Objects.requireNonNull(XContentFactory.xContentType(source)); + source = in.readString(); + if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); } updateAllTypes = in.readBoolean(); readTimeout(in); @@ -330,16 +320,24 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArrayNullable(indices); indicesOptions.writeIndicesOptions(out); out.writeOptionalString(type); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - out.writeBytesReference(source); - xContentType.writeTo(out); - } else if (xContentType.hasStringRepresentation()) { - out.writeString(source.utf8ToString()); - } else { - throw new IllegalStateException("cannot send [" + xContentType + "] to an older node"); - } + out.writeString(source); out.writeBoolean(updateAllTypes); writeTimeout(out); out.writeOptionalWriteable(concreteIndex); } + + private String convertToJsonIfNecessary(String source, XContentType xContentType) { + return convertToJsonIfNecessary(new BytesArray(source), xContentType); + } + + private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return source.utf8ToString(); + } + try { + return XContentHelper.convertToJson(source, false, xContentType); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to JSON", e); + } + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 9433259580b4d..d9ebf88fda6d7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -83,7 +83,7 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) .indices(concreteIndices).type(request.type()) .updateAllTypes(request.updateAllTypes()) - .source(request.source(), request.getXContentType()); + .source(request.source()); metaDataMappingService.putMapping(updateRequest, new ActionListener() { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 7af855b0262bf..56ae7c4648161 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; @@ -46,7 +45,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.io.UncheckedIOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -80,7 +79,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest> mappings = new HashMap<>(); + private Map mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -222,12 +221,12 @@ public Settings settings() { * * @param type The mapping type * @param source The mapping source - * @deprecated use {@link #mapping(String, BytesReference, XContentType)} + * @deprecated use {@link #mapping(String, String, XContentType)} */ @Deprecated public PutIndexTemplateRequest mapping(String type, String source) { XContentType xContentType = XContentFactory.xContentType(source); - return mapping(type, new BytesArray(source.getBytes(StandardCharsets.UTF_8)), xContentType); + return mapping(type, source, xContentType); } /** @@ -237,9 +236,9 @@ public PutIndexTemplateRequest mapping(String type, String source) { * @param source The mapping source * @param xContentType The type of content contained within the source */ - public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { Objects.requireNonNull(xContentType); - mappings.put(type, new Tuple<>(xContentType, source)); + mappings.put(type, convertToJsonIfNecessary(source, xContentType)); return this; } @@ -262,7 +261,11 @@ public String cause() { * @param source The mapping source */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { - return mapping(type, source.bytes(), source.contentType()); + try { + return mapping(type, source.string(), source.contentType()); + } catch (IOException e) { + throw new UncheckedIOException("failed to parse mapping", e); + } } /** @@ -279,7 +282,7 @@ public PutIndexTemplateRequest mapping(String type, Map source) try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.bytes(), XContentType.JSON); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -294,7 +297,7 @@ public PutIndexTemplateRequest mapping(String type, Object... source) { return this; } - public Map> mappings() { + public Map mappings() { return this.mappings; } @@ -529,20 +532,13 @@ public void readFrom(StreamInput in) throws IOException { create = in.readBoolean(); settings = readSettingsFromStream(in); int size = in.readVInt(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - for (int i = 0; i < size; i++) { - final String type = in.readString(); - final BytesReference source = in.readBytesReference(); - final XContentType xContentType = XContentType.readFrom(in); - mappings.put(type, new Tuple<>(xContentType, source)); - } - } else { - for (int i = 0; i < size; i++) { - final String type = in.readString(); - final BytesReference source = new BytesArray(in.readString().getBytes(StandardCharsets.UTF_8)); - final XContentType xContentType = XContentFactory.xContentType(source); - mappings.put(type, new Tuple<>(xContentType, source)); + for (int i = 0; i < size; i++) { + final String type = in.readString(); + String mappingSource = in.readString(); + if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + mappingSource = convertToJsonIfNecessary(mappingSource, XContentFactory.xContentType(mappingSource)); } + mappings.put(type, mappingSource); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { @@ -571,17 +567,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(create); writeSettingsToStream(settings, out); out.writeVInt(mappings.size()); - for (Map.Entry> entry : mappings.entrySet()) { + for (Map.Entry entry : mappings.entrySet()) { out.writeString(entry.getKey()); - Tuple value = entry.getValue(); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { - out.writeBytesReference(value.v2()); - value.v1().writeTo(out); - } else if (value.v1().hasStringRepresentation()) { - out.writeString(value.v2().utf8ToString()); - } else { - throw new IllegalStateException("cannot send [" + value.v1() + "] to an older node"); - } + out.writeString(entry.getValue()); } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { @@ -594,4 +582,19 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalVInt(version); } + + private String convertToJsonIfNecessary(String source, XContentType xContentType) { + return convertToJsonIfNecessary(new BytesArray(source), xContentType); + } + + private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return source.utf8ToString(); + } + try { + return XContentHelper.convertToJson(source, false, xContentType); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to JSON", e); + } + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index ab104fa6b91ef..66a9bba45e87b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -131,7 +131,7 @@ public PutIndexTemplateRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source - * @deprecated use {@link #addMapping(String, BytesReference, XContentType)} + * @deprecated use {@link #addMapping(String, String, XContentType)} */ @Deprecated public PutIndexTemplateRequestBuilder addMapping(String type, String source) { @@ -146,7 +146,7 @@ public PutIndexTemplateRequestBuilder addMapping(String type, String source) { * @param source The mapping source * @param xContentType The type/format of the source */ - public PutIndexTemplateRequestBuilder addMapping(String type, BytesReference source, XContentType xContentType) { + public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) { request.mapping(type, source, xContentType); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index c6bb1c353802b..9ed812abd4e4a 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; -import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; @@ -47,6 +46,7 @@ import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -83,7 +83,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; - private XContentType contentType = Requests.INDEX_CONTENT_TYPE; + private XContentType contentType; private String pipeline; @@ -290,12 +290,12 @@ public Map sourceAsMap() { } /** - * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}. + * Index the Map in {@link XContentType#JSON} format * * @param source The map to index */ public IndexRequest source(Map source) throws ElasticsearchGenerationException { - return source(source, contentType); + return source(source, XContentType.JSON); } /** @@ -320,9 +320,7 @@ public IndexRequest source(Map source, XContentType contentType) throws Elastics */ @Deprecated public IndexRequest source(String source) { - this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); - this.contentType = XContentFactory.xContentType(source); - return this; + return source(new BytesArray(source.getBytes(StandardCharsets.UTF_8)), XContentFactory.xContentType(source)); } /** @@ -332,18 +330,14 @@ public IndexRequest source(String source) { * or using the {@link #source(byte[], XContentType)}. */ public IndexRequest source(String source, XContentType xContentType) { - this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); - this.contentType = xContentType; - return this; + return source(new BytesArray(source.getBytes(StandardCharsets.UTF_8)), xContentType); } /** * Sets the content source to index. */ public IndexRequest source(XContentBuilder sourceBuilder) { - source = sourceBuilder.bytes(); - contentType = sourceBuilder.contentType(); - return this; + return source(sourceBuilder.bytes(), sourceBuilder.contentType()); } /** @@ -362,7 +356,8 @@ public IndexRequest source(Object... source) { throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)"); } try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentType sourceXContentType = contentType == null ? XContentType.JSON : contentType; + XContentBuilder builder = XContentFactory.contentBuilder(sourceXContentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -380,17 +375,16 @@ public IndexRequest source(Object... source) { */ @Deprecated public IndexRequest source(BytesReference source) { - this.source = source; - this.contentType = XContentFactory.xContentType(source); - return this; + return source(source, XContentFactory.xContentType(source)); + } /** * Sets the document to index in bytes form. */ public IndexRequest source(BytesReference source, XContentType xContentType) { - this.source = source; - this.contentType = xContentType; + this.source = Objects.requireNonNull(source); + this.contentType = Objects.requireNonNull(xContentType); return this; } @@ -421,9 +415,7 @@ public IndexRequest source(byte[] source, XContentType xContentType) { */ @Deprecated public IndexRequest source(byte[] source, int offset, int length) { - this.source = new BytesArray(source, offset, length); - this.contentType = XContentFactory.xContentType(source); - return this; + return source(new BytesArray(source, offset, length), XContentFactory.xContentType(source)); } /** @@ -435,9 +427,7 @@ public IndexRequest source(byte[] source, int offset, int length) { * @param length The length of the data */ public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { - this.source = new BytesArray(source, offset, length); - this.contentType = xContentType; - return this; + return source(new BytesArray(source, offset, length), xContentType); } /** diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index d595edcf4dd1f..ff97ae5cab3d8 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -35,7 +35,7 @@ public class PutPipelineRequest extends AcknowledgedRequest private String id; private BytesReference source; - private XContentType xContentType = XContentType.JSON; + private XContentType xContentType; @Deprecated public PutPipelineRequest(String id, BytesReference source) { diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index e22aa71c8bfd5..200b83ff02b96 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -46,7 +46,7 @@ public class SimulatePipelineRequest extends ActionRequest { private String id; private boolean verbose; private BytesReference source; - private XContentType xContentType = XContentType.JSON; + private XContentType xContentType; /** * Create a new request diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index b7d7cccb81853..5bba34904d0c8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -348,7 +348,7 @@ public Builder putMapping(String mappingType, CompressedXContent mappingSource) return this; } - public Builder putMapping(String mappingType, BytesReference mappingSource) throws IOException { + public Builder putMapping(String mappingType, String mappingSource) throws IOException { mappings.put(mappingType, new CompressedXContent(mappingSource)); return this; } @@ -457,7 +457,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t String mappingType = currentFieldName; Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); - builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).bytes()); + builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string()); } } } else if ("aliases".equals(currentFieldName)) { @@ -481,7 +481,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t Map mapping = parser.mapOrdered(); if (mapping.size() == 1) { String mappingType = mapping.keySet().iterator().next(); - BytesReference mappingSource = XContentFactory.jsonBuilder().map(mapping).bytes(); + String mappingSource = XContentFactory.jsonBuilder().map(mapping).string(); if (mappingSource == null) { // crap, no mapping source, warn? diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index fc09dbb00a7a0..0bde4a23b032f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,8 +53,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; @@ -63,10 +61,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -258,9 +253,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { List templateNames = new ArrayList<>(); - for (Map.Entry> entry : request.mappings().entrySet()) { - mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue().v2(), - entry.getValue().v1())); + for (Map.Entry entry : request.mappings().entrySet()) { + mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue())); } for (Map.Entry entry : request.customs().entrySet()) { @@ -271,14 +265,13 @@ public ClusterState execute(ClusterState currentState) throws Exception { for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); for (ObjectObjectCursor cursor : template.mappings()) { - BytesReference bytesReference = cursor.value.uncompressedReference(); - XContentType xContentType = XContentFactory.xContentType(bytesReference); + String mappingString = cursor.value.string(); if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), - MapperService.parseMapping(xContentRegistry, bytesReference, xContentType)); + MapperService.parseMapping(xContentRegistry, mappingString)); } else { mappings.put(cursor.key, - MapperService.parseMapping(xContentRegistry, bytesReference, xContentType)); + MapperService.parseMapping(xContentRegistry, mappingString)); } } // handle custom diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index a5bd516d621ee..e35fc7c837c46 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -30,8 +30,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; @@ -39,7 +37,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperParsingException; @@ -225,14 +222,13 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla templateBuilder.settings(request.settings); Map> mappingsForValidation = new HashMap<>(); - for (Map.Entry> entry : request.mappings.entrySet()) { + for (Map.Entry entry : request.mappings.entrySet()) { try { - templateBuilder.putMapping(entry.getKey(), entry.getValue().v2()); + templateBuilder.putMapping(entry.getKey(), entry.getValue()); } catch (Exception e) { throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage()); } - mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue().v2(), - entry.getValue().v1())); + mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue())); } dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false); @@ -320,7 +316,7 @@ public static class PutRequest { Integer version; List indexPatterns; Settings settings = Settings.Builder.EMPTY_SETTINGS; - Map> mappings = new HashMap<>(); + Map mappings = new HashMap<>(); List aliases = new ArrayList<>(); Map customs = new HashMap<>(); @@ -351,7 +347,7 @@ public PutRequest settings(Settings settings) { return this; } - public PutRequest mappings(Map> mappings) { + public PutRequest mappings(Map mappings) { this.mappings.putAll(mappings); return this; } @@ -366,8 +362,8 @@ public PutRequest customs(Map customs) { return this; } - public PutRequest putMapping(String mappingType, BytesReference mappingSource, XContentType xContentType) { - mappings.put(mappingType, new Tuple<>(xContentType, mappingSource)); + public PutRequest putMapping(String mappingType, String mappingSource) { + mappings.put(mappingType, mappingSource); return this; } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 8e7ae4f33a54f..c0032a4b6a452 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -259,9 +259,9 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt DocumentMapper existingMapper = mapperService.documentMapper(request.type()); if (MapperService.DEFAULT_MAPPING.equals(request.type())) { // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default - newMapper = mapperService.parse(request.type(), mappingUpdateSource, false, request.xContentType()); + newMapper = mapperService.parse(request.type(), mappingUpdateSource, false); } else { - newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null, request.xContentType()); + newMapper = mapperService.parse(request.type(), mappingUpdateSource, existingMapper == null); if (existingMapper != null) { // first, simulate: just call merge and ignore the result existingMapper.merge(newMapper.mapping(), request.updateAllTypes()); diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java index 059572848dc57..991dfccf7f994 100644 --- a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java +++ b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java @@ -146,15 +146,6 @@ public byte[] uncompressed() { } } - /** Return the uncompressed bytes as a {@link BytesReference} */ - public BytesReference uncompressedReference() { - try { - return CompressorFactory.uncompress(new BytesArray(bytes)); - } catch (IOException e) { - throw new IllegalStateException("Cannot decompress compressed bytes", e); - } - } - public String string() throws IOException { return new BytesRef(uncompressed()).utf8ToString(); } diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index e7f5f65c06bbe..f59340efcee55 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -971,7 +971,6 @@ public Builder loadFromSource(String source) { /** * Loads settings from the actual string content that represents them using the * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader - * Note only types that return {@code true} from {@link XContentType#hasStringRepresentation()} are supported */ public Builder loadFromSource(String source, XContentType xContentType) { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType); diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java index 1e4ae185115e2..5d8cb4918b2ea 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java +++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java @@ -79,8 +79,8 @@ public static SettingsLoader loaderFromSource(String source) { } /** - * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only types that return {@code true} from - * {@link XContentType#hasStringRepresentation()} are supported + * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only {@link XContentType#JSON} and + * {@link XContentType#YAML} are supported * * @param xContentType The content type * @return A settings loader. diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 94721cbc40a70..9256555c14b86 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -86,16 +86,40 @@ public interface XContentGenerator extends Closeable, Flushable { void writeBinary(byte[] value, int offset, int length) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #writeRawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, InputStream value) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + */ void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException; + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, BytesReference value) throws IOException; + /** + * Writes a raw field with the given bytes as the value + */ void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException; + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawValue(BytesReference value) throws IOException; + /** + * Writes a value with the source coming directly from the bytes + */ void writeRawValue(BytesReference value, XContentType xContentType) throws IOException; void copyCurrentStructure(XContentParser parser) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index ad3acfa4e8c6a..8e3c298270444 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Locale; +import java.util.Objects; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -58,11 +59,6 @@ public String shortName() { public XContent xContent() { return JsonXContent.jsonXContent; } - - @Override - public boolean hasStringRepresentation() { - return true; - } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -82,11 +78,6 @@ public String shortName() { public XContent xContent() { return SmileXContent.smileXContent; } - - @Override - public boolean hasStringRepresentation() { - return false; - } }, /** * A YAML based content type. @@ -106,11 +97,6 @@ public String shortName() { public XContent xContent() { return YamlXContent.yamlXContent; } - - @Override - public boolean hasStringRepresentation() { - return true; - } }, /** * A CBOR based content type. @@ -130,18 +116,13 @@ public String shortName() { public XContent xContent() { return CborXContent.cborXContent; } - - @Override - public boolean hasStringRepresentation() { - return false; - } }; /** * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a - * format query parameter. This method will return {@code null} if no match is found + * format query string parameter. This method will return {@code null} if no match is found */ public static XContentType fromMediaTypeOrFormat(String mediaType) { if (mediaType == null) { @@ -161,11 +142,11 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) { /** * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. - * The provided media type should not included any parameters. This method is suitable for parsing part of the {@code Content-Type} + * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type} * HTTP header. This method will return {@code null} if no match is found */ - public static XContentType fromMediaTypeStrict(String mediaType) { - final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT); + public static XContentType fromMediaType(String mediaType) { + final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT); for (XContentType type : values()) { if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) { return type; @@ -200,11 +181,6 @@ public String mediaType() { public abstract String mediaTypeWithoutParameters(); - /** - * Returns {@code true} if this {@link XContentType} can be represented as a String and is not a binary only format - */ - public abstract boolean hasStringRepresentation(); - public static XContentType readFrom(StreamInput in) throws IOException { int index = in.readVInt(); for (XContentType contentType : values()) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index a75466f42b4cb..a8c74101bff45 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -196,10 +196,6 @@ public CompressedXContent mappingSource() { return this.mappingSource; } - public XContentType mappingSourceXContentType() { - return XContentType.JSON; - } - public RootObjectMapper root() { return mapping.root; } @@ -263,7 +259,7 @@ public Map objectMappers() { // TODO this method looks like it is only used in tests... public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException { - return parse(SourceToParse.source(index, type, id, source, XContentFactory.xContentType(source))); + return parse(SourceToParse.source(index, type, id, source, XContentType.JSON)); } public ParsedDocument parse(SourceToParse source) throws MapperParsingException { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 8146e79574eae..be25775c1353e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -75,19 +74,13 @@ public Mapper.TypeParser.ParserContext parserContext(String type) { } public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { - return parse(type, source, null, null); + return parse(type, source, null); } - public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource, - XContentType xContentType) throws MapperParsingException { + public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { - Map root; - if (xContentType != null) { - root = XContentHelper.convertToMap(source.compressedReference(), true, xContentType).v2(); - } else { - root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); - } + Map root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2(); Tuple> t = extractMapping(type, root); type = t.v1(); mapping = t.v2(); @@ -169,7 +162,7 @@ private static String getRemainingFields(Map map) { private Tuple> extractMapping(String type, String source) throws MapperParsingException { Map root; - try (XContentParser parser = XContentFactory.xContent(source).createParser(xContentRegistry, source)) { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, source)) { root = parser.mapOrdered(); } catch (Exception e) { throw new MapperParsingException("failed to parse mapping definition", e); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index b8ca85cd1f168..41b29f3c67f4b 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -191,9 +191,11 @@ public DocumentMapperParser documentMapperParser() { return this.documentParser; } - public static Map parseMapping(NamedXContentRegistry xContentRegistry, BytesReference mappingSource, - XContentType xContentType) throws Exception { - try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, mappingSource)) { + /** + * Parses the mappings (formatted as JSON) into a map + */ + public static Map parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, mappingSource)) { return parser.map(); } } @@ -322,7 +324,7 @@ private synchronized Map internalMerge(Map Source [" @@ -618,9 +620,8 @@ private void checkPartitionedIndexConstraints(DocumentMapper newMapper) { } } - public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault, - XContentType xContentType) throws MapperParsingException { - return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null, xContentType); + public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException { + return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null); } public boolean hasMapping(String mappingType) { @@ -659,7 +660,7 @@ public DocumentMapperForType documentMapperWithAutoCreate(String type) { throw new TypeMissingException(index(), new IllegalStateException("trying to auto create mapping, but dynamic mapping is disabled"), type); } - mapper = parse(type, null, true, null); + mapper = parse(type, null, true); return new DocumentMapperForType(mapper, mapper.mapping()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 6a0129af7683e..2eb2fdeaa0333 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -241,8 +241,7 @@ protected void parseCreateField(ParseContext context, List field if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = context.sourceToParse().getXContentType() == null ? - XContentHelper.convertToMap(source, true) : + Tuple> mapTuple = XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType()); Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 3b8ddb113abd2..a8a983ecde832 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -60,8 +60,8 @@ private SourceToParse(Origin origin, String index, String type, String id, Bytes this.id = Objects.requireNonNull(id); // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays - this.source = new BytesArray(source.toBytesRef()); - this.xContentType = xContentType; + this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); + this.xContentType = Objects.requireNonNull(xContentType); } public Origin origin() { diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 90e7093d96ba5..7870a6e8668db 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Objects; /** * Encapsulates a pipeline's id and configuration as a blob @@ -76,9 +77,6 @@ void setXContentType(XContentType xContentType) { } PipelineConfiguration build() { - if (xContentType == null) { - xContentType = XContentFactory.xContentType(config); - } return new PipelineConfiguration(id, config, xContentType); } } @@ -91,9 +89,9 @@ PipelineConfiguration build() { private final XContentType xContentType; public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { - this.id = id; - this.config = config; - this.xContentType = xContentType; + this.id = Objects.requireNonNull(id); + this.config = Objects.requireNonNull(config); + this.xContentType = Objects.requireNonNull(xContentType); } public String getId() { diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index fc6d5f87f0cf8..bdc78c82dd503 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -66,15 +66,26 @@ public XContentBuilder newErrorBuilder() throws IOException { return newBuilder(request.getXContentType(), false); } + /** + * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following + * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format + * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map + * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls + * back to JSON. + */ @Override - public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { - XContentType contentType = XContentType.fromMediaTypeOrFormat(format); - if (contentType == null) { - if (xContentType != null) { - contentType = xContentType; + public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + // try to determine the response content type from the media type or the format query string parameter, with the format parameter + // taking precedence over the Accept header + XContentType responseContentType = XContentType.fromMediaTypeOrFormat(format); + if (responseContentType == null) { + if (requestContentType != null) { + // if there was a parsed content-type for the incoming request use that since no format was specified using the query + // string parameter or the HTTP Accept header + responseContentType = requestContentType; } else { - // default to JSON output - contentType = XContentType.JSON; + // default to JSON output when all else fails + responseContentType = XContentType.JSON; } } @@ -86,7 +97,7 @@ public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean u excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet()); } - XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes); + XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), bytesOutput(), includes, excludes); if (pretty) { builder.prettyPrint().lfAtEnd(); } @@ -122,11 +133,4 @@ public RestRequest request() { public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } - - @Override - public void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException { - sendResponse(new BytesRestResponse(restStatus, newErrorBuilder().startObject() - .field("error", errorMessage) - .endObject())); - } } diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java index 5abaf0d6c1316..7efc733f65133 100644 --- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java +++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import java.io.IOException; @@ -140,4 +141,10 @@ private static XContentBuilder build(RestChannel channel, RestStatus status, Exc builder.endObject(); return builder; } + + static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException { + return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject() + .field("error", errorMessage) + .endObject()); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index 91bbc0e9c6132..8c8346f0ef4b2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -48,9 +47,4 @@ public interface RestChannel { boolean detailedErrorsEnabled(); void sendResponse(RestResponse response); - - /** - * Sends a new simple error response with the error message included - */ - void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 73693c095de0e..c5724188f5ad0 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -174,8 +174,10 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont RestChannel responseChannel = channel; try { final int contentLength = request.hasContent() ? request.content().length() : 0; + assert contentLength >= 0 : "content length was negative, how is that possible?"; final RestHandler handler = getHandler(request); - if (checkForContentType(request, responseChannel, contentLength, handler)) { + + if (contentLength == 0 || checkForContentType(request, handler)) { if (canTripCircuitBreaker(request)) { inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); } else { @@ -184,6 +186,8 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont // iff we could reserve bytes for the request we need to send the response also over this channel responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); dispatchRequest(request, responseChannel, client, threadContext, handler); + } else { + sendContentTypeErrorMessage(request, responseChannel); } } catch (Exception e) { try { @@ -198,8 +202,8 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext, final RestHandler handler) throws Exception { - if (!checkRequestParameters(request, channel)) { - return; + if (checkRequestParameters(request, channel) == false) { + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled.")); } try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { for (String key : headersToCopy) { @@ -229,44 +233,37 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text */ - boolean checkForContentType(final RestRequest restRequest, final RestChannel channel, final int contentLength, - final RestHandler restHandler) { - if (contentLength > 0) { - if (restRequest.getXContentType() == null) { - if (restHandler != null && restHandler.supportsPlainText()) { - deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + - "in a supported format."); - } else if (isContentTypeRequired) { - sendContentTypeErrorMessage(restRequest, channel); - return false; - } else { - deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using" + - "the [Content-Type] header."); - XContentType xContentType = XContentFactory.xContentType(restRequest.content()); - if (xContentType == null) { - if (restHandler != null) { - if (restHandler.supportsPlainText()) { - deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + - "in a supported format."); - } else { - try { - channel.sendErrorResponse(NOT_ACCEPTABLE, "Could not determine content type from request body."); - } catch (IOException e) { - logger.warn("Failed to send response", e); - } - return false; - } + boolean checkForContentType(final RestRequest restRequest, final RestHandler restHandler) { + if (restRequest.getXContentType() == null) { + if (restHandler != null && restHandler.supportsPlainText()) { + // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can + // be removed! + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else if (isContentTypeRequired) { + return false; + } else { + deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using" + + "the [Content-Type] header."); + XContentType xContentType = XContentFactory.xContentType(restRequest.content()); + if (xContentType == null) { + if (restHandler != null) { + if (restHandler.supportsPlainText()) { + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else { + return false; } - } else { - restRequest.setxContentType(xContentType); } + } else { + restRequest.setXContentType(xContentType); } } } return true; } - private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) { + private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) throws IOException { final List contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); final String errorMessage; if (contentTypeHeader == null) { @@ -276,11 +273,7 @@ private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel ch Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; } - try { - channel.sendErrorResponse(NOT_ACCEPTABLE, errorMessage); - } catch (IOException e) { - logger.warn("Failed to send response", e); - } + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(NOT_ACCEPTABLE, errorMessage)); } /** @@ -291,11 +284,6 @@ boolean checkRequestParameters(final RestRequest request, final RestChannel chan // error_trace cannot be used when we disable detailed errors // we consume the error_trace parameter first to ensure that it is always consumed if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) { - try { - channel.sendErrorResponse(BAD_REQUEST, "error traces in responses are disabled."); - } catch (IOException e) { - logger.warn("Failed to send response", e); - } return false; } @@ -402,11 +390,6 @@ public void sendResponse(RestResponse response) { delegate.sendResponse(response); } - @Override - public void sendErrorResponse(RestStatus restStatus, String errorMessage) throws IOException { - delegate.sendErrorResponse(restStatus, errorMessage); - } - private void close() { // attempt to close once atomically if (closed.compareAndSet(false, true) == false) { diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index eced420b580ef..200f1f877b0a9 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -184,7 +184,7 @@ public final XContentType getXContentType() { * @deprecated this is only used to allow BWC with content-type detection */ @Deprecated - final void setxContentType(XContentType xContentType) { + final void setXContentType(XContentType xContentType) { this.xContentType.set(xContentType); } @@ -339,7 +339,7 @@ public final XContentParser contentParser() throws IOException { if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); } else if (xContentType.get() == null) { - throw new IllegalStateException("Content-Type must be provided"); + throw new IllegalStateException("no content-type has been set so we cannot create a parser"); } return xContentType.get().xContent().createParser(xContentRegistry, content); } @@ -485,11 +485,11 @@ private static XContentType parseContentType(List header) { final String[] splitMediaType = elements[0].split("/"); if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { - return XContentType.fromMediaTypeStrict(elements[0]); + return XContentType.fromMediaType(elements[0]); } else { throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); } } - return null; + throw new IllegalArgumentException("empty Content-Type header"); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 4b9bc29aff4a3..06fe67926455f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -64,7 +64,7 @@ public RestPutMappingAction(Settings settings, RestController controller) { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); putMappingRequest.type(request.param("type")); - putMappingRequest.source(request.content().utf8ToString(), request.getXContentType()); + putMappingRequest.source(request.content(), request.getXContentType()); putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java index f4b7ef86341f9..c0016ff0e26b0 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.XContentType; @@ -77,8 +76,7 @@ public static String parseStoredScript(BytesReference scriptAsBytes, XContentTyp // 3) just as is // In order to fetch the actual script in consistent manner this parsing logic is needed: // EMPTY is ok here because we never call namedObject, we're just copying structure. - try (XContentParser parser = xContentType == null ? XContentHelper.createParser(NamedXContentRegistry.EMPTY, scriptAsBytes) : - xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, scriptAsBytes); + try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, scriptAsBytes); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { parser.nextToken(); parser.nextToken(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java index 18ac79b09efa9..c9ce52a5d8a0c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -37,16 +37,14 @@ public void testSerialization() throws IOException { BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); request.mapping("my_type", bytesReference, XContentType.JSON); - assertEquals(XContentType.JSON, request.mappings().get("my_type").v1()); try (BytesStreamOutput output = new BytesStreamOutput()) { request.writeTo(output); try (StreamInput in = output.bytes().streamInput()) { CreateIndexRequest serialized = new CreateIndexRequest(); serialized.readFrom(in); - assertEquals(XContentType.JSON, serialized.mappings().get("my_type").v1()); assertEquals(request.index(), serialized.index()); - assertEquals(bytesReference, serialized.mappings().get("my_type").v2()); + assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type")); } } } @@ -59,10 +57,9 @@ public void testSerializationBwc() throws IOException { in.setVersion(version); CreateIndexRequest serialized = new CreateIndexRequest(); serialized.readFrom(in); - assertEquals(XContentType.JSON, serialized.mappings().get("my_type").v1()); assertEquals("foo", serialized.index()); BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); - assertEquals(bytesReference, serialized.mappings().get("my_type").v2()); + assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type")); try (BytesStreamOutput out = new BytesStreamOutput()) { out.setVersion(version); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index a9dde649ff673..4dc396323c048 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -19,16 +19,13 @@ package org.elasticsearch.action.admin.indices.mapping.get; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -37,7 +34,7 @@ public class GetFieldMappingsResponseTests extends ESTestCase { public void testSerialization() throws IOException { Map>> mappings = new HashMap<>(); - FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}"), XContentType.JSON); + FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}")); mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData))); GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); @@ -48,30 +45,8 @@ public void testSerialization() throws IOException { serialized.readFrom(in); FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field"); assertNotNull(metaData); - assertEquals(XContentType.JSON, metaData.getXContentType()); assertEquals(new BytesArray("{}"), metaData.getSource()); } } } - - public void testSerializationBwc() throws IOException { - final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEFZmllbGQIbXkgZmllbGQCe30="); - final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2, - Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED); - try (StreamInput in = StreamInput.wrap(data)) { - in.setVersion(version); - GetFieldMappingsResponse response = new GetFieldMappingsResponse(); - response.readFrom(in); - FieldMappingMetaData metaData = response.fieldMappings("index", "type", "field"); - assertNotNull(metaData); - assertEquals(XContentType.JSON, metaData.getXContentType()); - assertEquals(new BytesArray("{}"), metaData.getSource()); - - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.setVersion(version); - response.writeTo(out); - assertArrayEquals(data, out.bytes().toBytesRef().bytes); - } - } - } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 284dc19eca335..fd7f830e59230 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -21,9 +21,10 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.Index; @@ -78,7 +79,7 @@ public void testPutMappingRequestSerialization() throws IOException { PutMappingRequest request = new PutMappingRequest("foo"); String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); request.source(mapping, XContentType.YAML); - assertEquals(mapping, request.source().utf8ToString()); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source()); BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); request.writeTo(bytesStreamOutput); @@ -86,9 +87,8 @@ public void testPutMappingRequestSerialization() throws IOException { PutMappingRequest serialized = new PutMappingRequest(); serialized.readFrom(in); - BytesReference source = serialized.source(); - assertEquals(mapping, source.utf8ToString()); - assertEquals(XContentType.YAML, request.getXContentType()); + String source = serialized.source(); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source); } public void testSerializationBwc() throws IOException { @@ -100,14 +100,7 @@ public void testSerializationBwc() throws IOException { PutMappingRequest request = new PutMappingRequest(); request.readFrom(in); String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); - assertEquals(mapping, request.source().utf8ToString()); - assertEquals(XContentType.YAML, request.getXContentType()); - - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.setVersion(version); - request.writeTo(out); - assertArrayEquals(data, out.bytes().toBytesRef().bytes); - } + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source()); } } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index 5643e4ea27814..48598ecb2ecdb 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -26,12 +26,10 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService; import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; @@ -102,7 +100,7 @@ public void testIndexTemplateWithAliasNameEqualToTemplatePattern() { public void testIndexTemplateWithValidateEmptyMapping() throws Exception { PutRequest request = new PutRequest("api", "validate_template"); request.patterns(Collections.singletonList("validate_template")); - request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); + request.putMapping("type1", "{}"); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -115,7 +113,7 @@ public void testIndexTemplateWithValidateMapping() throws Exception { request.patterns(Collections.singletonList("te*")); request.putMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("field2").field("type", "text").field("analyzer", "custom_1").endObject() - .endObject().endObject().endObject().bytes(), XContentType.JSON); + .endObject().endObject().endObject().string()); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -126,7 +124,7 @@ public void testIndexTemplateWithValidateMapping() throws Exception { public void testBrokenMapping() throws Exception { PutRequest request = new PutRequest("api", "broken_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", new BytesArray("abcde"), XContentType.JSON); + request.putMapping("type1", "abcde"); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -137,7 +135,7 @@ public void testBrokenMapping() throws Exception { public void testBlankMapping() throws Exception { PutRequest request = new PutRequest("api", "blank_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); + request.putMapping("type1", "{}"); List errors = putTemplateDetail(request); assertThat(errors.size(), equalTo(1)); @@ -149,7 +147,7 @@ public void testAliasInvalidFilterInvalidJson() throws Exception { //invalid json: put index template fails PutRequest request = new PutRequest("api", "blank_mapping"); request.patterns(Collections.singletonList("te*")); - request.putMapping("type1", new BytesArray("{}"), XContentType.JSON); + request.putMapping("type1", "{}"); Set aliases = new HashSet<>(); aliases.add(new Alias("invalid_alias").filter("abcde")); request.aliases(aliases); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index e2cc3f74cc8ea..48b2ae79cf39c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -20,9 +20,9 @@ import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.ESTestCase; @@ -71,10 +71,11 @@ public void testPutIndexTemplateRequest510() throws IOException { public void testPutIndexTemplateRequestSerializationXContent() throws IOException { PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo"); - BytesReference mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); request.patterns(Collections.singletonList("foo")); request.mapping("bar", mapping, XContentType.YAML); - assertEquals(mapping, request.mappings().get("bar").v2()); + assertNotEquals(mapping, request.mappings().get("bar")); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar")); BytesStreamOutput out = new BytesStreamOutput(); request.writeTo(out); @@ -82,8 +83,7 @@ public void testPutIndexTemplateRequestSerializationXContent() throws IOExceptio StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); PutIndexTemplateRequest serialized = new PutIndexTemplateRequest(); serialized.readFrom(in); - assertEquals(mapping, serialized.mappings().get("bar").v2()); - assertEquals(XContentType.YAML, serialized.mappings().get("bar").v1()); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), serialized.mappings().get("bar")); } public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException { @@ -94,17 +94,11 @@ public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOExcep in.setVersion(version); PutIndexTemplateRequest request = new PutIndexTemplateRequest(); request.readFrom(in); - assertEquals(XContentType.YAML, request.mappings().get("bar").v1()); - BytesReference mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); - assertEquals(mapping, request.mappings().get("bar").v2()); + String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string(); + assertNotEquals(mapping, request.mappings().get("bar")); + assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar")); assertEquals("foo", request.name()); assertEquals("template", request.patterns().get(0)); - - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.setVersion(version); - request.writeTo(out); - assertArrayEquals(data, out.bytes().toBytesRef().bytes); - } } } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java index a6a270c57ff57..c3e544f9b2b84 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java @@ -21,16 +21,13 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.hamcrest.Matchers.equalTo; @@ -90,8 +87,7 @@ public void testMappingClusterStateUpdateDoesntChangeExistingIndices() throws Ex final ClusterService clusterService = getInstanceFromNode(ClusterService.class); // TODO - it will be nice to get a random mapping generator final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type"); - request.source(new BytesArray("{ \"properties\" { \"field\": { \"type\": \"string\" }}}".getBytes(StandardCharsets.UTF_8)), - XContentType.JSON); + request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}"); mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request)); assertThat(indexService.mapperService().documentMapper("type").mappingSource(), equalTo(currentMapping)); } @@ -102,8 +98,7 @@ public void testClusterStateIsNotChangedWithIdenticalMappings() throws Exception final MetaDataMappingService mappingService = getInstanceFromNode(MetaDataMappingService.class); final ClusterService clusterService = getInstanceFromNode(ClusterService.class); final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type"); - request.source(new BytesArray("{ \"properties\" { \"field\": { \"type\": \"string\" }}}".getBytes(StandardCharsets.UTF_8)), - XContentType.JSON); + request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}"); ClusterState result = mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request)) .resultingState; diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 97638b53e372b..3806b03ce48b6 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -717,7 +717,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { public void testFlushIsDisabledDuringTranslogRecovery() throws IOException { assertFalse(engine.isRecovering()); - ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), SOURCE, null); engine.index(indexForDoc(doc)); engine.close(); @@ -726,7 +726,7 @@ public void testFlushIsDisabledDuringTranslogRecovery() throws IOException { assertTrue(engine.isRecovering()); engine.recoverFromTranslog(); assertFalse(engine.isRecovering()); - doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null); + doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), SOURCE, null); engine.index(indexForDoc(doc)); engine.flush(); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index c907202dbd9c8..5543baff1a23b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -176,14 +176,13 @@ public void testMergeWithMap() throws Throwable { MapperService mapperService = indexService1.mapperService(); Map> mappings = new HashMap<>(); - mappings.put(MapperService.DEFAULT_MAPPING, - MapperService.parseMapping(xContentRegistry(), new BytesArray("{}"), XContentType.JSON)); + mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping(xContentRegistry(), "{}")); MapperException e = expectThrows(MapperParsingException.class, () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false)); assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: ")); mappings.clear(); - mappings.put("type1", MapperService.parseMapping(xContentRegistry(), new BytesArray("{}"), XContentType.JSON)); + mappings.put("type1", MapperService.parseMapping(xContentRegistry(), "{}")); e = expectThrows( MapperParsingException.class, () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java index 61e936b269e88..44484530856bc 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java @@ -39,7 +39,7 @@ public void testExceptionForCopyToInMultiFields() throws IOException { // first check that for newer versions we throw exception if copy_to is found withing multi field MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY); try { - mapperService.parse("type", new CompressedXContent(mapping.string()), true, mapping.contentType()); + mapperService.parse("type", new CompressedXContent(mapping.string()), true); fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field"); } catch (MapperParsingException e) { assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field.")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java index 644df1839f4d5..8c6ee8da04220 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java @@ -36,7 +36,7 @@ public void testExceptionForIncludeInAllInMultiFields() throws IOException { // first check that for newer versions we throw exception if include_in_all is found withing multi field MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY); Exception e = expectThrows(MapperParsingException.class, () -> - mapperService.parse("type", new CompressedXContent(mapping.string()), true, mapping.contentType())); + mapperService.parse("type", new CompressedXContent(mapping.string()), true)); assertEquals("include_in_all in multi fields is not allowed. Found the include_in_all in field [c] which is within a multi field.", e.getMessage()); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index aff1a9583149e..5803d7d957e1d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -113,11 +113,11 @@ public void testDefaultMappingAndNoMapping() throws Exception { .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper mapper = parser.parse("my_type", null, defaultMapping, null); + DocumentMapper mapper = parser.parse("my_type", null, defaultMapping); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); try { - mapper = parser.parse(null, null, defaultMapping, null); + mapper = parser.parse(null, null, defaultMapping); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); fail(); @@ -125,7 +125,7 @@ public void testDefaultMappingAndNoMapping() throws Exception { // all is well } try { - mapper = parser.parse(null, new CompressedXContent("{}"), defaultMapping, XContentType.JSON); + mapper = parser.parse(null, new CompressedXContent("{}"), defaultMapping); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); fail(); @@ -145,7 +145,7 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception { .endObject().endObject().string(); DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser() - .parse("my_type", new CompressedXContent(mapping), defaultMapping, XContentType.JSON); + .parse("my_type", new CompressedXContent(mapping), defaultMapping); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(true)); } diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 2d82b1ec025ac..eb9b17172325b 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -337,7 +337,7 @@ public void testBrokenMapping() throws Exception { MapperParsingException e = expectThrows( MapperParsingException.class, () -> client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", new BytesArray("abcde"), XContentType.JSON) + .addMapping("type1", "abcde", XContentType.JSON) .get()); assertThat(e.getMessage(), containsString("Failed to parse mapping ")); @@ -374,8 +374,7 @@ public void testIndexTemplateWithAliases() throws Exception { client().admin().indices().preparePutTemplate("template_with_aliases") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", new BytesArray("{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}"), - XContentType.JSON) + .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON) .addAlias(new Alias("simple_alias")) .addAlias(new Alias("templated_alias-{index}")) .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}")) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 009616281e663..90d7983be01ca 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -95,7 +95,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder Date: Thu, 26 Jan 2017 13:03:16 -0500 Subject: [PATCH 14/34] change onOrAfter to after for now --- .../admin/cluster/storedscripts/PutStoredScriptRequest.java | 4 ++-- .../action/admin/indices/create/CreateIndexRequest.java | 2 +- .../action/admin/indices/mapping/put/PutMappingRequest.java | 2 +- .../admin/indices/template/put/PutIndexTemplateRequest.java | 2 +- .../java/org/elasticsearch/action/index/IndexRequest.java | 4 ++-- .../org/elasticsearch/action/ingest/PutPipelineRequest.java | 4 ++-- .../elasticsearch/action/ingest/SimulatePipelineRequest.java | 4 ++-- .../elasticsearch/action/termvectors/TermVectorsRequest.java | 4 ++-- .../elasticsearch/index/query/MoreLikeThisQueryBuilder.java | 4 ++-- .../java/org/elasticsearch/ingest/PipelineConfiguration.java | 4 ++-- .../org/elasticsearch/percolator/PercolateQueryBuilder.java | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 2603bca73c482..9b620ec8e7d66 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -125,7 +125,7 @@ public void readFrom(StreamInput in) throws IOException { scriptLang = in.readString(); id = in.readOptionalString(); script = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType = XContentType.readFrom(in); } else { xContentType = XContentFactory.xContentType(script); @@ -138,7 +138,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(scriptLang); out.writeOptionalString(id); out.writeBytesReference(script); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index b0d0ffc5c8663..6515fd122054b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -538,7 +538,7 @@ public void readFrom(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { final String type = in.readString(); String source = in.readString(); - if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); } mappings.put(type, source); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index e796c70144d32..039464160cef6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -306,7 +306,7 @@ public void readFrom(StreamInput in) throws IOException { indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); source = in.readString(); - if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); } updateAllTypes = in.readBoolean(); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 56ae7c4648161..c35260c36ceea 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -535,7 +535,7 @@ public void readFrom(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { final String type = in.readString(); String mappingSource = in.readString(); - if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported mappingSource = convertToJsonIfNecessary(mappingSource, XContentFactory.xContentType(mappingSource)); } mappings.put(type, mappingSource); diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 9ed812abd4e4a..1a4fb19372199 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -559,7 +559,7 @@ public void readFrom(StreamInput in) throws IOException { pipeline = in.readOptionalString(); isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting contentType = XContentType.readFrom(in); } else { contentType = XContentFactory.xContentType(source); @@ -592,7 +592,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(pipeline); out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting contentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index ff97ae5cab3d8..8174e272cc90f 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -73,7 +73,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); source = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType = XContentType.readFrom(in); } else { xContentType = XContentFactory.xContentType(source); @@ -85,7 +85,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 200b83ff02b96..362d2e7ab98ca 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -103,7 +103,7 @@ public void readFrom(StreamInput in) throws IOException { id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType = XContentType.readFrom(in); } else { xContentType = XContentFactory.xContentType(source); @@ -116,7 +116,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index 292be11022031..544b14523bc70 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -499,7 +499,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.readBoolean()) { doc = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType = XContentType.readFrom(in); } else { xContentType = XContentFactory.xContentType(doc); @@ -544,7 +544,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index ea61edbbf9cad..3bbdfd798ad7c 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -228,7 +228,7 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) type = in.readOptionalString(); if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType = XContentType.readFrom(in); } else { xContentType = XContentFactory.xContentType(doc); @@ -250,7 +250,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } else { diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 7870a6e8668db..538a05f16b716 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -122,7 +122,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in)); } else { final String id = in.readString(); @@ -139,7 +139,7 @@ public static Diff readDiffFrom(StreamInput in) throws IO public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting xContentType.writeTo(out); } } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 90d7983be01ca..97a65a29adab2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -184,7 +184,7 @@ public PercolateQueryBuilder(String field, String documentType, String indexedDo } document = in.readOptionalBytesReference(); if (document != null) { - if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting documentXContentType = XContentType.readFrom(in); } else { documentXContentType = XContentFactory.xContentType(document); @@ -210,7 +210,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeOptionalBytesReference(document); - if (document != null && out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) { + if (document != null && out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting documentXContentType.writeTo(out); } } From 2391d2d30cb0548bb56fbab59321178d7187fa4a Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 13:06:19 -0500 Subject: [PATCH 15/34] rollback change to bwc test version --- qa/backwards-5.0/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle index 82fd14c1e15f4..fceeac283fc3a 100644 --- a/qa/backwards-5.0/build.gradle +++ b/qa/backwards-5.0/build.gradle @@ -38,7 +38,7 @@ integTest { cluster { numNodes = 4 numBwcNodes = 2 - bwcVersion = "6.0.0-alpha1-SNAPSHOT" // TODO switch back to 5.3 and remove blacklist once backported + bwcVersion = "5.3.0-SNAPSHOT" setting 'logger.org.elasticsearch', 'DEBUG' } } From 19cdb2758fe88a2f0269bc9303acc4fb22d9fee8 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 13:09:06 -0500 Subject: [PATCH 16/34] put mapping request can take bytes so we can pass from rest layer --- .../admin/indices/mapping/put/PutMappingRequest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 039464160cef6..5278aee9d0400 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -288,6 +288,15 @@ public PutMappingRequest source(String mappingSource, XContentType xContentType) return this; } + /** + * The mapping source definition. + */ + public PutMappingRequest source(BytesReference mappingSource, XContentType xContentType) { + Objects.requireNonNull(xContentType); + this.source = convertToJsonIfNecessary(mappingSource, xContentType); + return this; + } + /** True if all fields that span multiple types should be updated, false otherwise */ public boolean updateAllTypes() { return updateAllTypes; From 0264168b96aaf2df7b1ced0a1106e8a1a7090272 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 13:41:40 -0500 Subject: [PATCH 17/34] fix reindex parsing of content-type --- .../index/reindex/remote/RemoteScrollableHitSource.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java index 54b8ac8c61f85..bc58c3e51f32e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -21,6 +21,7 @@ import org.apache.http.ContentTooLongException; import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -161,7 +162,8 @@ public void onSuccess(org.elasticsearch.client.Response response) { InputStream content = responseEntity.getContent(); XContentType xContentType = null; if (responseEntity.getContentType() != null) { - xContentType = XContentType.fromMediaType(responseEntity.getContentType().getValue()); + final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType(); + xContentType = XContentType.fromMediaType(mimeType); } if (xContentType == null) { try { From b18cae71b1303edca424f881891d8bf8e893c670 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 13:47:45 -0500 Subject: [PATCH 18/34] docs updates --- docs/reference/api-conventions.asciidoc | 24 +++++++++++++++++-- .../migration/migrate_6_0/rest.asciidoc | 6 +++++ docs/reference/modules/http.asciidoc | 3 +++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 712a7969a0517..7e8ed1c1c816f 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -670,7 +670,27 @@ The response looks like: For libraries that don't accept a request body for non-POST requests, you can pass the request body as the `source` query string parameter -instead. +instead. When using this method, the `source_content_type` parameter +should also be passed with a media type value that indicates the format +of the source, such as `application/json`. + +[float] +=== Content-Type auto-detection + +deprecated[5.3.0, Provide the proper Content-Type header] + +The content sent in a request body or using the `source` query string +parameter is inspected to automatically determine the content type +(JSON, YAML, SMILE, or CBOR). + +A strict mode can be enabled which disables auto-detection and requires +that all requests with a body have a Content-Type header that maps to +a supported format. To enabled this strict mode, add the following +setting to the `elasticsearch.yml` file: + + http.content_type.required: true + +The default value is `false`. [[url-access-control]] == URL-based access control @@ -682,7 +702,7 @@ the choice of specifying an index in the URL and on each individual request within the request body. This can make URL-based access control challenging. To prevent the user from overriding the index which has been specified in the -URL, add this setting to the `config.yml` file: +URL, add this setting to the `elasticsearch.yml` file: rest.action.multi.allow_explicit_index: false diff --git a/docs/reference/migration/migrate_6_0/rest.asciidoc b/docs/reference/migration/migrate_6_0/rest.asciidoc index 1999ff6517126..f7b52b914daf3 100644 --- a/docs/reference/migration/migrate_6_0/rest.asciidoc +++ b/docs/reference/migration/migrate_6_0/rest.asciidoc @@ -13,6 +13,12 @@ has been removed in Elasticsearch 6.0.0. In previous versions of Elasticsearch, documents were allowed to contain duplicate keys. Elasticsearch 6.0.0 enforces that all keys are unique. This applies to all content types: JSON, CBOR, Yaml and Smile. +==== Content-Type Auto-detection + +In previous versions of Elasticsearch, having a proper Content-Type for the data in a request was not enforced. +Elasticsearch 6.0.0 enforces that all requests with a body must have a supported Content-Type and this type will +be used when parsing the data. + ==== Boolean API parameters All REST APIs parameters (both request parameters and JSON body) support providing boolean "false" as the diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index 065c91349c25a..3682234f5e6fb 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -100,6 +100,9 @@ simple message will be returned. Defaults to `true` |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`. +|`http.content_type.required`|Enables or disables strict checking and usage of +the `Content-Type` header for all requests with content, defaults to `false`. + |======================================================================= It also uses the common From 7e94108a6049b4a2b80a3ffa05267abcfd1f59ea Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 14:02:19 -0500 Subject: [PATCH 19/34] fix index request serialization for reindex --- .../action/index/IndexRequest.java | 11 +++++++++-- .../action/index/IndexRequestTests.java | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 1a4fb19372199..fdc656269ea06 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -560,7 +560,9 @@ public void readFrom(StreamInput in) throws IOException { isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting - contentType = XContentType.readFrom(in); + if (in.readBoolean()) { + contentType = XContentType.readFrom(in); + } } else { contentType = XContentFactory.xContentType(source); } @@ -593,7 +595,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting - contentType.writeTo(out); + if (contentType != null) { + out.writeBoolean(true); + contentType.writeTo(out); + } else { + out.writeBoolean(false); + } } } diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index 39463d9f19866..fb1991bcd91ea 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -185,4 +185,21 @@ public void testIndexRequestXContentSerializationBwc() throws IOException { // don't test writing to earlier versions since output differs due to no timestamp } } + + // reindex makes use of index requests without a source so this needs to be handled + public void testSerializationOfEmptyRequestWorks() throws IOException { + IndexRequest request = new IndexRequest("index", "type"); + assertNull(request.getContentType()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + + try (StreamInput in = out.bytes().streamInput()) { + IndexRequest serialized = new IndexRequest(); + serialized.readFrom(in); + assertNull(request.getContentType()); + assertEquals("index", request.index()); + assertEquals("type", request.type()); + } + } + } } From 65bfe9e1770094ab291c7d430c731591e997383b Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 16:47:22 -0500 Subject: [PATCH 20/34] make reindex work with the removal of the default value in IndexRequest --- .../action/index/IndexRequest.java | 18 +++++--- .../reindex/ClientScrollableHitSource.java | 6 +++ .../index/reindex/ScrollableHitSource.java | 16 ++++++- .../index/reindex/TransportReindexAction.java | 26 ++++++++++- .../reindex/remote/RemoteResponseParsers.java | 45 +++++++++++++------ .../remote/RemoteScrollableHitSource.java | 5 ++- .../reindex/AsyncBulkByScrollActionTests.java | 3 +- 7 files changed, 91 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index fdc656269ea06..9f87d4002ffe8 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -140,7 +140,9 @@ public ActionRequestValidationException validate() { if (source == null) { validationException = addValidationError("source is missing", validationException); } - + if (contentType == null) { + validationException = addValidationError("content type is missing", validationException); + } final long resolvedVersion = resolveVersionDefaults(); if (opType() == OpType.CREATE) { if (versionType != VersionType.INTERNAL) { @@ -187,11 +189,11 @@ public XContentType getContentType() { } /** - * Sets the content type. This will be used when generating a document from user provided objects (like Map) and when parsing the + * Sets the content type. This will be used when generating a document from user provided objects (like Maps) and when parsing the * source at index time */ public IndexRequest contentType(XContentType contentType) { - this.contentType = contentType; + this.contentType = Objects.requireNonNull(contentType); return this; } @@ -320,7 +322,7 @@ public IndexRequest source(Map source, XContentType contentType) throws Elastics */ @Deprecated public IndexRequest source(String source) { - return source(new BytesArray(source.getBytes(StandardCharsets.UTF_8)), XContentFactory.xContentType(source)); + return source(new BytesArray(source), XContentFactory.xContentType(source)); } /** @@ -330,7 +332,7 @@ public IndexRequest source(String source) { * or using the {@link #source(byte[], XContentType)}. */ public IndexRequest source(String source, XContentType xContentType) { - return source(new BytesArray(source.getBytes(StandardCharsets.UTF_8)), xContentType); + return source(new BytesArray(source), xContentType); } /** @@ -356,8 +358,10 @@ public IndexRequest source(Object... source) { throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)"); } try { - XContentType sourceXContentType = contentType == null ? XContentType.JSON : contentType; - XContentBuilder builder = XContentFactory.contentBuilder(sourceXContentType); + if (contentType == null) { + contentType = XContentType.JSON; + } + XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java index 8d3583a077d5f..60d22ed9ce9f3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java @@ -37,6 +37,8 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.search.SearchHit; @@ -232,6 +234,10 @@ public BytesReference getSource() { return source; } + @Override + public XContentType getXContentType() { + return XContentFactory.xContentType(source); + } @Override public long getVersion() { return delegate.getVersion(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java index c48858fcf2052..92ef140f4b01a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource; import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; @@ -81,7 +82,7 @@ public final void startNextScroll(TimeValue extraKeepAlive, Consumer o }); } protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer onResponse); - + @Override public final void close() { String scrollId = this.scrollId.get(); @@ -190,6 +191,10 @@ public interface Hit { * all. */ @Nullable BytesReference getSource(); + /** + * The content type of the hit source. Returns null if the source didn't come back from the search. + */ + @Nullable XContentType getXContentType(); /** * The document id of the parent of the hit if there is a parent or null if there isn't. */ @@ -211,6 +216,7 @@ public static class BasicHit implements Hit { private final long version; private BytesReference source; + private XContentType xContentType; private String parent; private String routing; @@ -246,8 +252,14 @@ public BytesReference getSource() { return source; } - public BasicHit setSource(BytesReference source) { + @Override + public XContentType getXContentType() { + return xContentType; + } + + public BasicHit setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; return this; } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 4e0562668f8c7..968d80ce31369 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -55,6 +55,10 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; @@ -66,6 +70,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -315,7 +321,24 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) // id and source always come from the found doc. Scripts can change them but they operate on the index request. index.id(doc.getId()); - index.source(doc.getSource()); + + // the source xcontent type and destination could be different + final XContentType sourceXContentType = doc.getXContentType(); + final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); + if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { + // we need to convert + try (XContentParser parser = sourceXContentType.xContent().createParser(NamedXContentRegistry.EMPTY, doc.getSource()); + XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent())) { + parser.nextToken(); + builder.copyCurrentStructure(parser); + index.source(builder.bytes(), builder.contentType()); + } catch (IOException e) { + throw new UncheckedIOException("failed to convert hit from " + sourceXContentType + " to " + + mainRequestXContentType, e); + } + } else { + index.source(doc.getSource(), doc.getXContentType()); + } /* * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass @@ -323,7 +346,6 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) */ index.routing(mainRequest.getDestination().routing()); index.parent(mainRequest.getDestination().parent()); - index.contentType(mainRequest.getDestination().getContentType()); index.setPipeline(mainRequest.getDestination().getPipeline()); // OpType is synthesized from version so it is handled when we copy version above. diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java index 8b9dc7b8de7f5..634634fd9e6ab 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; @@ -29,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; @@ -37,6 +39,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import java.util.function.BiFunction; import static java.util.Collections.emptyList; @@ -54,7 +57,7 @@ private RemoteResponseParsers() {} /** * Parser for an individual {@code hit} element. */ - public static final ConstructingObjectParser HIT_PARSER = + public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { int i = 0; String index = (String) a[i++]; @@ -68,15 +71,16 @@ private RemoteResponseParsers() {} HIT_PARSER.declareString(constructorArg(), new ParseField("_type")); HIT_PARSER.declareString(constructorArg(), new ParseField("_id")); HIT_PARSER.declareLong(optionalConstructorArg(), new ParseField("_version")); - HIT_PARSER.declareObject(BasicHit::setSource, (p, s) -> { + HIT_PARSER.declareObject(((basicHit, tuple) -> basicHit.setSource(tuple.v1(), tuple.v2())), (p, s) -> { try { /* * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for * now this should do. */ - try (XContentBuilder b = JsonXContent.contentBuilder()) { + try (XContentBuilder b = XContentBuilder.builder(s.getXContentType().xContent())) { b.copyCurrentStructure(p); - return b.bytes(); + // a hack but this lets us get the right xcontent type to go with the source + return new Tuple<>(b.bytes(), s.getXContentType()); } } catch (IOException e) { throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e); @@ -92,7 +96,7 @@ class Fields { String routing; String parent; } - ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); + ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); hit.setParent(fields.parent); @@ -105,7 +109,7 @@ class Fields { /** * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}. */ - public static final ConstructingObjectParser HITS_PARSER = + public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); static { HITS_PARSER.declareLong(constructorArg(), new ParseField("total")); @@ -115,7 +119,7 @@ class Fields { /** * Parser for {@code failed} shards in the {@code _shards} elements. */ - public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = + public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = new ConstructingObjectParser<>("failure", true, a -> { int i = 0; String index = (String) a[i++]; @@ -148,7 +152,7 @@ class Fields { * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it * parses to an empty list. */ - public static final ConstructingObjectParser, Void> SHARDS_PARSER = + public static final ConstructingObjectParser, ResponseContext> SHARDS_PARSER = new ConstructingObjectParser<>("_shards", true, a -> { @SuppressWarnings("unchecked") List failures = (List) a[0]; @@ -159,7 +163,7 @@ class Fields { SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures")); } - public static final ConstructingObjectParser RESPONSE_PARSER = + public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>("search_response", true, a -> { int i = 0; Throwable catastrophicFailure = (Throwable) a[i++]; @@ -198,9 +202,9 @@ class Fields { * Collects stuff about Throwables and attempts to rebuild them. */ public static class ThrowableBuilder { - public static final BiFunction PARSER; + public static final BiFunction PARSER; static { - ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); + ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); PARSER = parser.andThen(ThrowableBuilder::build); parser.declareString(ThrowableBuilder::setType, new ParseField("type")); parser.declareString(ThrowableBuilder::setReason, new ParseField("reason")); @@ -219,7 +223,7 @@ public static class ThrowableBuilder { public Throwable build() { Throwable t = buildWithoutCause(); - if (causedBy != null) { + if (causedBy != null) { t.initCause(causedBy); } return t; @@ -264,12 +268,25 @@ public void setCausedBy(Throwable causedBy) { /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ - public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( "/", true, a -> (Version) a[0]); static { - ConstructingObjectParser versionParser = new ConstructingObjectParser<>( + ConstructingObjectParser versionParser = new ConstructingObjectParser<>( "version", true, a -> Version.fromString((String) a[0])); versionParser.declareString(constructorArg(), new ParseField("number")); MAIN_ACTION_PARSER.declareObject(constructorArg(), versionParser, new ParseField("version")); } + + static final class ResponseContext { + + private final XContentType xContentType; + + ResponseContext(XContentType xContentType) { + this.xContentType = Objects.requireNonNull(xContentType); + } + + XContentType getXContentType() { + return this.xContentType; + } + } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java index bc58c3e51f32e..0368a8b3797d2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -45,6 +45,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.index.reindex.remote.RemoteResponseParsers.ResponseContext; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -142,7 +143,7 @@ protected void cleanup() { } private void execute(String method, String uri, Map params, HttpEntity entity, - BiFunction parser, Consumer listener) { + BiFunction parser, Consumer listener) { // Preserve the thread context so headers survive after the call java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); class RetryHelper extends AbstractRunnable { @@ -178,7 +179,7 @@ public void onSuccess(org.elasticsearch.client.Response response) { // EMPTY is safe here because we don't call namedObject try (XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) { - parsedResponse = parser.apply(xContentParser, null); + parsedResponse = parser.apply(xContentParser, new ResponseContext(xContentType)); } catch (ParsingException e) { /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it * is totally wrong and we're probably not talking to Elasticsearch. */ diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 23192f083d987..71f01d8262593 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -61,6 +61,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; @@ -383,7 +384,7 @@ protected AbstractAsyncBulkByScrollAction.RequestWrapper buildRequest(Hit doc } }; ScrollableHitSource.BasicHit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0); - hit.setSource(new BytesArray("{}")); + hit.setSource(new BytesArray("{}"), XContentType.JSON); ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null); simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response); ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get()); From 1d1d47e2b6c878f0ef386867cea5a605d1e39b23 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 16:52:50 -0500 Subject: [PATCH 21/34] add content type message to test --- .../java/org/elasticsearch/action/bulk/BulkRequestTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index a2463a41656ff..e33caf30047c1 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -202,7 +202,8 @@ public void testBulkNoSource() throws Exception { assertThat(validate.validationErrors(), not(empty())); assertThat(validate.validationErrors(), contains( "script or doc is missing", - "source is missing")); + "source is missing", + "content type is missing")); } public void testCannotAddNullRequests() throws Exception { From 7357e4e39a937c636bc617fe09245714315eca5d Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 17:25:28 -0500 Subject: [PATCH 22/34] fix testing broken mapping to use valid json --- .../elasticsearch/indices/template/SimpleIndexTemplateIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index eb9b17172325b..cb6b8b752bcf3 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -337,7 +337,7 @@ public void testBrokenMapping() throws Exception { MapperParsingException e = expectThrows( MapperParsingException.class, () -> client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) - .addMapping("type1", "abcde", XContentType.JSON) + .addMapping("type1", "{\"foo\": \"abcde\"}", XContentType.JSON) .get()); assertThat(e.getMessage(), containsString("Failed to parse mapping ")); From 88a1a569bcadd859b018d8a5d545adf71c857314 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 26 Jan 2017 18:23:01 -0500 Subject: [PATCH 23/34] ensure we consume the source_content_type parameter before checking if source is null --- core/src/main/java/org/elasticsearch/rest/RestRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index 200f1f877b0a9..ed2929ca5bf83 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -406,10 +406,11 @@ public final Tuple contentOrSourceParam() { } return new Tuple<>(xContentType.get(), content()); } + String source = param("source"); + String typeParam = param("source_content_type"); if (source != null) { BytesArray bytes = new BytesArray(source); - String typeParam = param("source_content_type"); final XContentType xContentType; if (typeParam != null) { xContentType = parseContentType(Collections.singletonList(typeParam)); From 842806ad209d3e11e839b59d6863d99d0edd9409 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 27 Jan 2017 07:03:18 -0500 Subject: [PATCH 24/34] handle backwards compatibility in rest test client --- .../elasticsearch/test/rest/yaml/ClientYamlTestClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index 0324fedd57a98..3bc810eccce87 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -122,7 +122,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params if (supportedMethods.contains("GET") && RandomizedTest.rarely()) { logger.debug("sending the request body as source param with GET method"); queryStringParams.put("source", body); - queryStringParams.put("source_content_type", ContentType.APPLICATION_JSON.toString()); + if (esVersion.after(Version.V_5_3_0_UNRELEASED)) { // TODO make onOrAfter with backport + queryStringParams.put("source_content_type", ContentType.APPLICATION_JSON.toString()); + } requestMethod = "GET"; } else { requestMethod = RandomizedTest.randomFrom(supportedMethods); From 26c6df8f8d300e345faec676e72fc8b39ef7a20e Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 27 Jan 2017 10:29:07 -0500 Subject: [PATCH 25/34] modify rest test framework to ignore deprecated missing source_content_type header when running a mixed cluster --- .../test/rest/yaml/ClientYamlTestClient.java | 12 +++++++----- .../yaml/ClientYamlTestExecutionContext.java | 4 ++-- .../test/rest/yaml/ClientYamlTestResponse.java | 16 ++++++++++++++-- .../yaml/ClientYamlTestResponseException.java | 5 +++-- .../rest/yaml/ESClientYamlSuiteTestCase.java | 17 +++++++++++------ 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index 3bc810eccce87..aa51efaa33319 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -55,13 +55,15 @@ public class ClientYamlTestClient { private final ClientYamlSuiteRestSpec restSpec; private final RestClient restClient; private final Version esVersion; + private final Map hostVersionMap; public ClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, - Version esVersion) throws IOException { + Version esVersion, Map hostVersionMap) throws IOException { assert hosts.size() > 0; this.restSpec = restSpec; this.restClient = restClient; this.esVersion = esVersion; + this.hostVersionMap = hostVersionMap; } public Version getEsVersion() { @@ -86,9 +88,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params // And everything else is a url parameter! try { Response response = restClient.performRequest(method, path, queryStringParams, entity); - return new ClientYamlTestResponse(response); + return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost())); } catch(ResponseException e) { - throw new ClientYamlTestResponseException(e); + throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost())); } } @@ -171,9 +173,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params logger.debug("calling api [{}]", apiName); try { Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders); - return new ClientYamlTestResponse(response); + return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost())); } catch(ResponseException e) { - throw new ClientYamlTestResponseException(e); + throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost())); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 8215105887ded..27653a13f2940 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -71,9 +71,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params throw e; } finally { // if we hit a bad exception the response is null - Object repsponseBody = response != null ? response.getBody() : null; + Object responseBody = response != null ? response.getBody() : null; //we always stash the last response body - stash.stashValue("body", repsponseBody); + stash.stashValue("body", responseBody); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java index 481ae752d05f0..d1027156c74b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java @@ -21,6 +21,7 @@ import org.apache.http.Header; import org.apache.http.client.methods.HttpHead; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.XContentType; @@ -37,10 +38,12 @@ public class ClientYamlTestResponse { private final Response response; private final String body; + private final Version nodeVersion; private ObjectPath parsedResponse; - ClientYamlTestResponse(Response response) throws IOException { + ClientYamlTestResponse(Response response, Version version) throws IOException { this.response = response; + this.nodeVersion = version; if (response.getEntity() != null) { try { this.body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); @@ -80,7 +83,16 @@ public List getWarningHeaders() { List warningHeaders = new ArrayList<>(); for (Header header : response.getHeaders()) { if (header.getName().equals("Warning")) { - warningHeaders.add(header.getValue()); + if (nodeVersion.after(Version.V_5_3_0_UNRELEASED) && response.getRequestLine().getMethod().equals("GET") + && response.getRequestLine().getUri().contains("source") + && response.getRequestLine().getUri().contains("source_content_type") == false && header.getValue().startsWith( + "Deprecated use of the [source] parameter without the [source_content_type] parameter.")) { + // this is because we do not sent the source content type header when the node is 5.3.0 or below + // TODO remove this when we bump versions + } else { + warningHeaders.add(header.getValue()); + } + } } return warningHeaders; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java index 7d983d480296b..48a7ee578d51a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.rest.yaml; +import org.elasticsearch.Version; import org.elasticsearch.client.ResponseException; import java.io.IOException; @@ -32,10 +33,10 @@ public class ClientYamlTestResponseException extends IOException { private final ClientYamlTestResponse restTestResponse; private final ResponseException responseException; - ClientYamlTestResponseException(ResponseException responseException) throws IOException { + ClientYamlTestResponseException(ResponseException responseException, Version version) throws IOException { super(responseException); this.responseException = responseException; - this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse()); + this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse(), version); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index a45d92c9699de..411f657329d8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -53,6 +53,7 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -125,6 +126,7 @@ public void initAndResetContext() throws IOException { validateSpec(restSpec); List hosts = getClusterHosts(); RestClient restClient = client(); + Tuple> versionMapTuple = readVersionsFromInfo(restClient, hosts.size()); Version esVersion; try { Tuple versionVersionTuple = readVersionsFromCatNodes(restClient); @@ -135,13 +137,14 @@ public void initAndResetContext() throws IOException { } catch (ResponseException ex) { if (ex.getResponse().getStatusLine().getStatusCode() == 403) { logger.warn("Fallback to simple info '/' request, _cat/nodes is not authorized"); - esVersion = readVersionsFromInfo(restClient, hosts.size()); + esVersion = versionMapTuple.v1(); logger.info("initializing yaml client, minimum es version: [{}] hosts: {}", esVersion, hosts); } else { throw ex; } } - ClientYamlTestClient clientYamlTestClient = new ClientYamlTestClient(restSpec, restClient, hosts, esVersion); + ClientYamlTestClient clientYamlTestClient = + new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, versionMapTuple.v2()); restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient); adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient); String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); @@ -290,7 +293,7 @@ public static void clearStatic() { private static Tuple readVersionsFromCatNodes(RestClient restClient) throws IOException { // we simply go to the _cat/nodes API and parse all versions in the cluster Response response = restClient.performRequest("GET", "/_cat/nodes", Collections.singletonMap("h", "version,master")); - ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); + ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT); String nodesCatResponse = restTestResponse.getBodyAsString(); String[] split = nodesCatResponse.split("\n"); Version version = null; @@ -313,12 +316,13 @@ private static Tuple readVersionsFromCatNodes(RestClient restC return new Tuple<>(version, masterVersion); } - private static Version readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException { + private static Tuple> readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException { Version version = null; + Map hostVersionMap = new HashMap<>(); for (int i = 0; i < numHosts; i++) { //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster Response response = restClient.performRequest("GET", "/"); - ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); + ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT); Object latestVersion = restTestResponse.evaluate("version.number"); if (latestVersion == null) { throw new RuntimeException("elasticsearch version not found in the response"); @@ -329,8 +333,9 @@ private static Version readVersionsFromInfo(RestClient restClient, int numHosts) } else if (version.onOrAfter(currentVersion)) { version = currentVersion; } + hostVersionMap.put(response.getHost(), currentVersion); } - return version; + return new Tuple<>(version, Collections.unmodifiableMap(hostVersionMap)); } public void test() throws IOException { From a441458ee0427d15badc0e97ab9f91ee665a63cd Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 27 Jan 2017 14:21:12 -0500 Subject: [PATCH 26/34] address more feedback --- .../put/PutRepositoryRequest.java | 2 +- .../RestoreSnapshotRequestBuilder.java | 4 +- .../indices/create/CreateIndexRequest.java | 31 +++++--------- .../mapping/put/PutMappingRequest.java | 30 ++++--------- .../settings/put/UpdateSettingsRequest.java | 2 +- .../template/put/PutIndexTemplateRequest.java | 42 +++++++++---------- .../put/PutIndexTemplateRequestBuilder.java | 6 +-- .../action/index/IndexRequest.java | 35 +++++++++------- .../action/index/IndexRequestBuilder.java | 16 +++++-- .../action/ingest/PutPipelineRequest.java | 7 ++++ .../SimulatePipelineRequestBuilder.java | 17 ++++++++ .../action/update/UpdateRequest.java | 25 ++++++++--- .../action/update/UpdateRequestBuilder.java | 27 +++++++++--- .../common/xcontent/XContentHelper.java | 27 +++--------- .../ingest/PipelineConfiguration.java | 8 +--- .../action/IndicesRequestIT.java | 13 +++--- .../action/ListenerActionIT.java | 3 +- .../action/bulk/BulkProcessorIT.java | 10 +++-- .../action/bulk/BulkRequestTests.java | 4 +- .../action/bulk/BulkWithUpdatesIT.java | 32 +++++++------- .../OldIndexBackwardsCompatibilityIT.java | 3 +- .../elasticsearch/cluster/NoMasterNodeIT.java | 3 +- .../index/WaitUntilRefreshIT.java | 8 ++-- .../mapper/DynamicMappingDisabledTests.java | 3 +- .../breaker/CircuitBreakerServiceIT.java | 3 +- .../elasticsearch/ingest/IngestClientIT.java | 3 +- .../ingest/PipelineExecutionServiceTests.java | 5 ++- .../elasticsearch/routing/AliasRoutingIT.java | 3 +- .../routing/SimpleRoutingIT.java | 18 ++++---- .../aggregations/bucket/ChildrenIT.java | 3 +- 30 files changed, 218 insertions(+), 175 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index 2e78b51084df8..f0f8d50b4c1c9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -142,7 +142,7 @@ public PutRepositoryRequest settings(Settings.Builder settings) { /** * Sets the repository settings. * - * @param source repository settings in json, yaml or properties format + * @param source repository settings in json or yaml format * @return this request * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 4b220ab677392..807e238724364 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -154,7 +154,7 @@ public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * @@ -169,7 +169,7 @@ public RestoreSnapshotRequestBuilder setSettings(String source) { } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format *

* See repository documentation for more information. * diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 6515fd122054b..34cc694cec613 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -45,7 +45,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -240,8 +239,12 @@ public CreateIndexRequest mapping(String type, BytesReference source, XContentTy throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } Objects.requireNonNull(xContentType); - mappings.put(type, convertToJsonIfNecessary(source, xContentType)); - return this; + try { + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert to json", e); + } } /** @@ -354,14 +357,14 @@ public CreateIndexRequest alias(Alias alias) { */ @Deprecated public CreateIndexRequest source(String source) { - return source(source.getBytes(StandardCharsets.UTF_8)); + return source(new BytesArray(source)); } /** * Sets the settings and mappings as a single source. */ public CreateIndexRequest source(String source, XContentType xContentType) { - return source(source.getBytes(StandardCharsets.UTF_8), xContentType); + return source(new BytesArray(source), xContentType); } /** @@ -539,7 +542,8 @@ public void readFrom(StreamInput in) throws IOException { final String type = in.readString(); String source = in.readString(); if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport - source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); + // we do not know the content type that comes from earlier versions so we autodetect and convert + source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); } mappings.put(type, source); } @@ -581,19 +585,4 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(updateAllTypes); waitForActiveShards.writeTo(out); } - - private String convertToJsonIfNecessary(String source, XContentType xContentType) { - return convertToJsonIfNecessary(new BytesArray(source), xContentType); - } - - private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { - if (xContentType == XContentType.JSON) { - return source.utf8ToString(); - } - try { - return XContentHelper.convertToJson(source, false, xContentType); - } catch (IOException e) { - throw new UncheckedIOException("failed to convert source to JSON", e); - } - } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 5278aee9d0400..400701f91ca2f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -283,9 +283,7 @@ public PutMappingRequest source(String mappingSource) { * The mapping source definition. */ public PutMappingRequest source(String mappingSource, XContentType xContentType) { - Objects.requireNonNull(xContentType); - this.source = convertToJsonIfNecessary(mappingSource, xContentType); - return this; + return source(new BytesArray(mappingSource), xContentType); } /** @@ -293,8 +291,12 @@ public PutMappingRequest source(String mappingSource, XContentType xContentType) */ public PutMappingRequest source(BytesReference mappingSource, XContentType xContentType) { Objects.requireNonNull(xContentType); - this.source = convertToJsonIfNecessary(mappingSource, xContentType); - return this; + try { + this.source = XContentHelper.convertToJson(mappingSource, false, false, xContentType); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to json", e); + } } /** True if all fields that span multiple types should be updated, false otherwise */ @@ -316,7 +318,8 @@ public void readFrom(StreamInput in) throws IOException { type = in.readOptionalString(); source = in.readString(); if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported - source = convertToJsonIfNecessary(source, XContentFactory.xContentType(source)); + // we do not know the format from earlier versions so convert if necessary + source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); } updateAllTypes = in.readBoolean(); readTimeout(in); @@ -334,19 +337,4 @@ public void writeTo(StreamOutput out) throws IOException { writeTimeout(out); out.writeOptionalWriteable(concreteIndex); } - - private String convertToJsonIfNecessary(String source, XContentType xContentType) { - return convertToJsonIfNecessary(new BytesArray(source), xContentType); - } - - private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { - if (xContentType == XContentType.JSON) { - return source.utf8ToString(); - } - try { - return XContentHelper.convertToJson(source, false, xContentType); - } catch (IOException e) { - throw new UncheckedIOException("failed to convert source to JSON", e); - } - } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index df1d4d9f46fc2..f7fbdf7667a36 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -156,7 +156,7 @@ public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) { } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) */ @SuppressWarnings("unchecked") public UpdateSettingsRequest settings(Map source) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index c35260c36ceea..b0c13540dfa0b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -199,7 +199,7 @@ public PutIndexTemplateRequest settings(String source, XContentType xContentType } /** - * The settings to create the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json or yaml format). */ public PutIndexTemplateRequest settings(Map source) { try { @@ -237,9 +237,7 @@ public PutIndexTemplateRequest mapping(String type, String source) { * @param xContentType The type of content contained within the source */ public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - mappings.put(type, convertToJsonIfNecessary(source, xContentType)); - return this; + return mapping(type, new BytesArray(source), xContentType); } /** @@ -261,10 +259,23 @@ public String cause() { * @param source The mapping source */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { + return mapping(type, source.bytes(), source.contentType()); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); try { - return mapping(type, source.string(), source.contentType()); + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; } catch (IOException e) { - throw new UncheckedIOException("failed to parse mapping", e); + throw new UncheckedIOException("failed to convert source to json", e); } } @@ -536,7 +547,9 @@ public void readFrom(StreamInput in) throws IOException { final String type = in.readString(); String mappingSource = in.readString(); if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported - mappingSource = convertToJsonIfNecessary(mappingSource, XContentFactory.xContentType(mappingSource)); + // we do not know the incoming type so convert it if needed + mappingSource = + XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource)); } mappings.put(type, mappingSource); } @@ -582,19 +595,4 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalVInt(version); } - - private String convertToJsonIfNecessary(String source, XContentType xContentType) { - return convertToJsonIfNecessary(new BytesArray(source), xContentType); - } - - private String convertToJsonIfNecessary(BytesReference source, XContentType xContentType) { - if (xContentType == XContentType.JSON) { - return source.utf8ToString(); - } - try { - return XContentHelper.convertToJson(source, false, xContentType); - } catch (IOException e) { - throw new UncheckedIOException("failed to convert source to JSON", e); - } - } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index 66a9bba45e87b..e7ca25e51191b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -101,7 +101,7 @@ public PutIndexTemplateRequestBuilder setSettings(Settings.Builder settings) { } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) * @deprecated use {@link #setSettings(String, XContentType)} */ @Deprecated @@ -111,7 +111,7 @@ public PutIndexTemplateRequestBuilder setSettings(String source) { } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) */ public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) { request.settings(source, xContentType); @@ -119,7 +119,7 @@ public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xC } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) */ public PutIndexTemplateRequestBuilder setSettings(Map source) { request.settings(source); diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 9f87d4002ffe8..b6ff97c693415 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; @@ -188,15 +189,6 @@ public XContentType getContentType() { return contentType; } - /** - * Sets the content type. This will be used when generating a document from user provided objects (like Maps) and when parsing the - * source at index time - */ - public IndexRequest contentType(XContentType contentType) { - this.contentType = Objects.requireNonNull(contentType); - return this; - } - /** * The type of the indexed document. */ @@ -292,12 +284,12 @@ public Map sourceAsMap() { } /** - * Index the Map in {@link XContentType#JSON} format + * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format * * @param source The map to index */ public IndexRequest source(Map source) throws ElasticsearchGenerationException { - return source(source, XContentType.JSON); + return source(source, Requests.INDEX_CONTENT_TYPE); } /** @@ -349,8 +341,22 @@ public IndexRequest source(XContentBuilder sourceBuilder) { * number. Also the first argument in each pair (the field name) must have a * valid String representation. *

+ * @deprecated use {@link #source(XContentType, Object...)} to be specific about your desired content type */ + @Deprecated public IndexRequest source(Object... source) { + return source(Requests.INDEX_CONTENT_TYPE, source); + } + + /** + * Sets the content source to index. + *

+ * Note: the number of objects passed to this method as varargs must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation. + *

+ */ + public IndexRequest source(XContentType xContentType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -358,10 +364,7 @@ public IndexRequest source(Object... source) { throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)"); } try { - if (contentType == null) { - contentType = XContentType.JSON; - } - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = XContentFactory.contentBuilder(xContentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -565,7 +568,7 @@ public void readFrom(StreamInput in) throws IOException { autoGeneratedTimestamp = in.readLong(); if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting if (in.readBoolean()) { - contentType = XContentType.readFrom(in); + contentType = XContentType.readFrom(in); } } else { contentType = XContentFactory.xContentType(source); diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 7e249aba23278..cd5a0c8ffaeee 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -204,17 +205,24 @@ public IndexRequestBuilder setSource(byte[] source, int offset, int length, XCon * number. Also the first argument in each pair (the field name) must have a * valid String representation. *

+ * @deprecated use {@link #setSource(XContentType, Object...)} instead to be explicit about the content type */ + @Deprecated public IndexRequestBuilder setSource(Object... source) { - request.source(source); + request.source(Requests.INDEX_CONTENT_TYPE, source); return this; } /** - * The content type that will be used to generate a document from user provided objects (like Map). + * Constructs a simple document with a field name and value pairs. + *

+ * Note: the number of objects passed as varargs to this method must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation. + *

*/ - public IndexRequestBuilder setContentType(XContentType contentType) { - request.contentType(contentType); + public IndexRequestBuilder setSource(XContentType xContentType, Object... source) { + request.source(xContentType, source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 8174e272cc90f..a28950b24c189 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -37,11 +37,18 @@ public class PutPipelineRequest extends AcknowledgedRequest private BytesReference source; private XContentType xContentType; + /** + * Create a new pipeline request + * @deprecated use {@link #PutPipelineRequest(String, BytesReference, XContentType)} to avoid content type auto-detection + */ @Deprecated public PutPipelineRequest(String id, BytesReference source) { this(id, source, XContentFactory.xContentType(source)); } + /** + * Create a new pipeline request with the id and source along with the content type of the source + */ public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { this.id = Objects.requireNonNull(id); this.source = Objects.requireNonNull(source); diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java index 287a1c7217a94..bb5d0e4e40003 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -26,25 +26,42 @@ public class SimulatePipelineRequestBuilder extends ActionRequestBuilder { + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action) { super(client, action, new SimulatePipelineRequest()); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + * @deprecated use {@link #SimulatePipelineRequestBuilder(ElasticsearchClient, SimulatePipelineAction, BytesReference, XContentType)} to + * avoid content type auto-detection on the source bytes + */ @Deprecated public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) { super(client, action, new SimulatePipelineRequest(source)); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source, XContentType xContentType) { super(client, action, new SimulatePipelineRequest(source, xContentType)); } + /** + * Set the id for the pipeline to simulate + */ public SimulatePipelineRequestBuilder setId(String id) { request.setId(id); return this; } + /** + * Enable or disable verbose mode + */ public SimulatePipelineRequestBuilder setVerbose(boolean verbose) { request.setVerbose(verbose); return this; diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 84b0c4157c74e..0cba05e721a72 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -610,17 +611,20 @@ public UpdateRequest doc(byte[] source, int offset, int length, XContentType xCo /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. + * @deprecated use {@link #doc(XContentType, Object...)} to be explicit about content type */ + @Deprecated public UpdateRequest doc(Object... source) { - safeDoc().source(source); + safeDoc().source(Requests.INDEX_CONTENT_TYPE, source); return this; } /** - * Sets the doc to use for updates when a script is not specified. + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. */ - public UpdateRequest doc(String field, Object value) { - safeDoc().source(field, value); + public UpdateRequest doc(XContentType xContentType, Object... source) { + safeDoc().source(xContentType, source); return this; } @@ -725,9 +729,20 @@ public UpdateRequest upsert(byte[] source, int offset, int length, XContentType /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. + * @deprecated use {@link #upsert(XContentType, Object...)} to be explicit about content type */ + @Deprecated public UpdateRequest upsert(Object... source) { - safeUpsertRequest().source(source); + safeUpsertRequest().source(Requests.INDEX_CONTENT_TYPE, source); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequest upsert(XContentType xContentType, Object... source) { + safeUpsertRequest().source(xContentType, source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index ae33ad6e40986..f1be0a05e6a9e 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; @@ -276,10 +277,13 @@ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XConte } /** - * Sets the doc to use for updates when a script is not specified. + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. + * @deprecated use {@link #setDoc(XContentType, Object...)} to be specific about content type */ - public UpdateRequestBuilder setDoc(String field, Object value) { - request.doc(field, value); + @Deprecated + public UpdateRequestBuilder setDoc(Object... source) { + request.doc(source); return this; } @@ -287,8 +291,8 @@ public UpdateRequestBuilder setDoc(String field, Object value) { * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. */ - public UpdateRequestBuilder setDoc(Object... source) { - request.doc(source); + public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) { + request.doc(xContentType, source); return this; } @@ -382,9 +386,20 @@ public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XCo /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. + * @deprecated use {@link #setUpsert(XContentType, Object...)} to be specific about the desired content type */ + @Deprecated public UpdateRequestBuilder setUpsert(Object... source) { - request.upsert(source); + request.upsert(Requests.INDEX_CONTENT_TYPE, source); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) { + request.upsert(xContentType, source); return this; } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 46965e2f1e737..9fc71b6838cd7 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -63,25 +63,7 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry @Deprecated public static Tuple> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { - try { - final XContentType contentType; - InputStream input; - Compressor compressor = CompressorFactory.compressor(bytes); - if (compressor != null) { - InputStream compressedStreamInput = compressor.streamInput(bytes.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - contentType = XContentFactory.xContentType(compressedStreamInput); - input = compressedStreamInput; - } else { - contentType = XContentFactory.xContentType(bytes); - input = bytes.streamInput(); - } - return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); - } catch (IOException e) { - throw new ElasticsearchParseException("Failed to parse content to map", e); - } + return convertToMap(bytes, ordered, null); } /** @@ -90,6 +72,7 @@ public static Tuple> convertToMap(BytesReferen public static Tuple> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) throws ElasticsearchParseException { try { + final XContentType contentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -101,7 +84,8 @@ public static Tuple> convertToMap(BytesReferen } else { input = bytes.streamInput(); } - return new Tuple<>(Objects.requireNonNull(xContentType), convertToMap(XContentFactory.xContent(xContentType), input, ordered)); + contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), input, ordered)); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); } @@ -150,10 +134,9 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, X public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType) throws IOException { + Objects.requireNonNull(xContentType); if (xContentType == XContentType.JSON && !reformatJson) { return bytes.utf8ToString(); - } else if (xContentType == null) { - throw new IllegalArgumentException("the xcontent type must be provided"); } // It is safe to use EMPTY here because this never uses namedObject diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 538a05f16b716..3e874a0fd1d7b 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -49,8 +49,7 @@ public final class PipelineConfiguration extends AbstractDiffable { XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); - builder.setConfig(contentBuilder.bytes()); - builder.setXContentType(contentBuilder.contentType()); + builder.setConfig(contentBuilder.bytes(), contentBuilder.contentType()); }, new ParseField("config"), ObjectParser.ValueType.OBJECT); } @@ -68,11 +67,8 @@ void setId(String id) { this.id = id; } - void setConfig(BytesReference config) { + void setConfig(BytesReference config, XContentType xContentType) { this.config = config; - } - - void setXContentType(XContentType xContentType) { this.xContentType = xContentType; } diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java index 791501373e6dd..cca9d73b78f16 100644 --- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java @@ -75,6 +75,7 @@ import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; @@ -203,7 +204,8 @@ public void testIndex() { String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"}; interceptTransportActions(indexShardActions); - IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id").source("field", "value"); + IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id") + .source(Requests.INDEX_CONTENT_TYPE, "field", "value"); internalCluster().coordOnlyNodeClient().index(indexRequest).actionGet(); clearInterceptedActions(); @@ -228,7 +230,7 @@ public void testUpdate() { String indexOrAlias = randomIndexOrAlias(); client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get(); - UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet(); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); @@ -242,7 +244,8 @@ public void testUpdateUpsert() { interceptTransportActions(updateShardActions); String indexOrAlias = randomIndexOrAlias(); - UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert("field", "value").doc("field1", "value1"); + UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert(Requests.INDEX_CONTENT_TYPE, "field", "value") + .doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet(); assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult()); @@ -275,7 +278,7 @@ public void testBulk() { int numIndexRequests = iterations(1, 10); for (int i = 0; i < numIndexRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value")); + bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); indices.add(indexOrAlias); } int numDeleteRequests = iterations(1, 10); @@ -287,7 +290,7 @@ public void testBulk() { int numUpdateRequests = iterations(1, 10); for (int i = 0; i < numUpdateRequests; i++) { String indexOrAlias = randomIndexOrAlias(); - bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1")); + bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1")); indices.add(indexOrAlias); } diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java index 60e4b4d26224f..d0e8d997a1bb9 100644 --- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java +++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.test.ESIntegTestCase; @@ -38,7 +39,7 @@ public void testThreadedListeners() throws Throwable { IndexRequest request = new IndexRequest("test", "type", "1"); if (randomBoolean()) { // set the source, without it, we will have a verification failure - request.source("field1", "value1"); + request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); } client.index(request, new ActionListener() { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java index 6bac7c2f8a436..7e608815c43c4 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -254,11 +255,13 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception for (int i = 1; i <= numDocs; i++) { if (randomBoolean()) { testDocs++; - processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source("field", "value")); + processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")); multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs)); } else { testReadOnlyDocs++; - processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source("field", "value")); + processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)) + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")); } } } @@ -296,7 +299,8 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) { MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet(); for (int i = 1; i <= numDocs; i++) { - processor.add(new IndexRequest("test", "test", Integer.toString(i)).source("field", randomRealisticUnicodeOfLengthBetween(1, 30))); + processor.add(new IndexRequest("test", "test", Integer.toString(i)) + .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30))); multiGetRequestBuilder.add("test", "test", Integer.toString(i)); } return multiGetRequestBuilder; diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index e33caf30047c1..a35a82ff096eb 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -120,8 +120,8 @@ public void testBulkAllowExplicitIndex() throws Exception { public void testBulkAddIterable() { BulkRequest bulkRequest = Requests.bulkRequest(); List requests = new ArrayList<>(); - requests.add(new IndexRequest("test", "test", "id").source("field", "value")); - requests.add(new UpdateRequest("test", "test", "id").doc("field", "value")); + requests.add(new IndexRequest("test", "test", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value")); + requests.add(new UpdateRequest("test", "test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")); requests.add(new DeleteRequest("test", "test", "id")); bulkRequest.add(requests); assertThat(bulkRequest.requests().size(), equalTo(3)); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index e87107b7fc00d..b31426e3978dd 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -210,9 +211,9 @@ public void testBulkVersioning() throws Exception { assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L)); bulkResponse = client().prepareBulk() - .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc("field", "2")) - .add(client().prepareUpdate("test", "type", "2").setDoc("field", "2")) - .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc("field", "3")).get(); + .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2")) + .add(client().prepareUpdate("test", "type", "2").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2")) + .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3")).get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L)); @@ -236,9 +237,9 @@ public void testBulkVersioning() throws Exception { bulkResponse = client().prepareBulk() .add(client().prepareUpdate("test", "type", "e1") - .setDoc("field", "2").setVersion(10)) // INTERNAL + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2").setVersion(10)) // INTERNAL .add(client().prepareUpdate("test", "type", "e1") - .setDoc("field", "3").setVersion(13).setVersionType(VersionType.INTERNAL)) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3").setVersion(13).setVersionType(VersionType.INTERNAL)) .get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); @@ -681,7 +682,8 @@ public void testFailingVersionedUpdatedOnBulk() throws Exception { return; } BulkRequestBuilder requestBuilder = client().prepareBulk(); - requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID)); + requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1) + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID)); responses[threadID] = requestBuilder.get(); }); @@ -769,10 +771,10 @@ private static String indexOrAlias() { public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{ createIndex("bulkindex1", "bulkindex2"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1")) - .add(new IndexRequest("bulkindex2", "index2_type", "1").source("text", "hallo2")) - .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2")) - .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar")) + bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) + .add(new IndexRequest("bulkindex2", "index2_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) + .add(new IndexRequest("bulkindex2", "index2_type").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2")) + .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex2", "index2_type", "3")) .setRefreshPolicy(RefreshPolicy.IMMEDIATE); @@ -795,8 +797,8 @@ public void testFailedRequestsOnClosedIndex() throws Exception { assertAcked(client().admin().indices().prepareClose("bulkindex1")); BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE); - bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1")) - .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar")) + bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1")) + .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar")) .add(new DeleteRequest("bulkindex1", "index1_type", "1")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); @@ -811,8 +813,10 @@ public void testFailedRequestsOnClosedIndex() throws Exception { // issue 9821 public void testInvalidIndexNamesCorrectOpType() { BulkResponse bulkResponse = client().prepareBulk() - .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1").setSource("field", 1)) - .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1").setDoc("field", randomInt())) + .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1") + .setSource(Requests.INDEX_CONTENT_TYPE, "field", 1)) + .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt())) .add(client().prepareDelete().setIndex("INVALID.NAME").setType("type1").setId("1")).get(); assertThat(bulkResponse.getItems().length, is(3)); assertThat(bulkResponse.getItems()[0].getOpType(), is(OpType.INDEX)); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java index 20add0530d828..76e9a4eb1053c 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.RecoverySource; @@ -388,7 +389,7 @@ void assertRealtimeGetWorks(String indexName) { SearchHit hit = searchReq.get().getHits().getAt(0); String docId = hit.getId(); // foo is new, it is not a field in the generated index - client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get(); + client().prepareUpdate(indexName, "doc", docId).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "bar").get(); GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get(); Map source = getRsp.getSourceAsMap(); assertThat(source, Matchers.hasKey("foo")); diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 73598c3effe4a..c799969a3c8e5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -236,7 +237,7 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { TimeValue timeout = TimeValue.timeValueMillis(200); long now = System.currentTimeMillis(); try { - client().prepareUpdate("test1", "type1", "1").setDoc("field", "value2").setTimeout(timeout).get(); + client().prepareUpdate("test1", "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout).get(); fail("Expected ClusterBlockException"); } catch (ClusterBlockException e) { assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50)); diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java index 0f72e72f6a25f..ca089e6eb83ba 100644 --- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java +++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; @@ -95,14 +96,15 @@ public void testUpdate() throws InterruptedException, ExecutionException { assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); // Update with RefreshPolicy.WAIT_UNTIL - UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + UpdateResponse update = client().prepareUpdate("test", "test", "1") + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .get(); assertEquals(2, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); // Upsert with RefreshPolicy.WAIT_UNTIL - update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat") + update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat") .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); assertEquals(1, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); @@ -125,7 +127,7 @@ public void testBulk() { // Update by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz")); + bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz")); assertBulkSuccess(bulk.get()); assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java index 346b441470609..46749e792ed7d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.update.UpdateHelper; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -110,7 +111,7 @@ public static void destroyThreadPool() { public void testDynamicDisabled() { IndexRequest request = new IndexRequest("index", "type", "1"); - request.source("foo", 3); + request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(request); final AtomicBoolean onFailureCalled = new AtomicBoolean(); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index aa960594dedc3..42b34609fe140 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -468,7 +469,7 @@ public void testLimitsRequestSize() throws Exception { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)); - indexRequest.source("field", "value", "num", i); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i); bulkRequest.add(indexRequest); } diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 1882031f3b90c..2b59f0d421cb9 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -145,7 +146,7 @@ public void testBulkWithIngestFailures() throws Exception { BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numRequests; i++) { IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id"); - indexRequest.source("field", "value", "fail", i % 2 == 0); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0); bulkRequest.add(indexRequest); } diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java index 3d409c61d5964..7ad445e301370 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentType; @@ -274,7 +275,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception { } } else { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId); - indexRequest.source("field1", "value1"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); request = indexRequest; numIndexRequests++; } @@ -304,7 +305,7 @@ public void testBulkRequestExecution() throws Exception { int numRequest = scaledRandomIntBetween(8, 64); for (int i = 0; i < numRequest; i++) { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId); - indexRequest.source("field1", "value1"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); bulkRequest.add(indexRequest); } diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index 621ade9e31de9..dc8b6b54464e4 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.Requests; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -64,7 +65,7 @@ public void testAliasCrudRouting() throws Exception { logger.info("--> updating with id [1] and routing through alias"); client().prepareUpdate("alias0", "type1", "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()) - .setDoc("field", "value2") + .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2") .execute().actionGet(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("alias0", "type1", "1").execute().actionGet().isExists(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 2490134db4e85..b027a13e867e1 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -207,13 +207,13 @@ public void testRequiredRoutingCrudApis() throws Exception { } try { - client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field", "value2").execute().actionGet(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").execute().actionGet(); fail("update with missing routing when routing is required should fail"); } catch(ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); } - client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc("field", "value2").execute().actionGet(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); client().admin().indices().prepareRefresh().execute().actionGet(); for (int i = 0; i < 5; i++) { @@ -254,7 +254,7 @@ public void testRequiredRoutingBulk() throws Exception { { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1") - .source("field", "value")).execute().actionGet(); + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -269,12 +269,13 @@ public void testRequiredRoutingBulk() throws Exception { { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").routing("0") - .source("field", "value")).execute().actionGet(); + .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } { - BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2")) + BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1") + .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2")) .execute().actionGet(); assertThat(bulkResponse.getItems().length, equalTo(1)); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -289,7 +290,8 @@ public void testRequiredRoutingBulk() throws Exception { } { - BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2") + BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1") + .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2") .routing("0")).execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(false)); } @@ -369,12 +371,12 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { } UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0") - .setDoc("field1", "value1").get(); + .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.getVersion(), equalTo(2L)); try { - client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field1", "value1").get(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); fail(); } catch (RoutingMissingException e) { assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index a4a6afa208dff..2268826100478 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -268,7 +269,7 @@ public void testWithDeletes() throws Exception { */ UpdateResponse updateResponse = client().prepareUpdate(indexName, "child", idToUpdate) .setParent("1") - .setDoc("count", 1) + .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1) .setDetectNoop(false) .get(); assertThat(updateResponse.getVersion(), greaterThan(1L)); From e208082e1ea6e47641cccbce3d14826a266f24c9 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 30 Jan 2017 09:12:03 -0500 Subject: [PATCH 27/34] address non-reindex comments --- .../indices/create/CreateIndexRequest.java | 15 +++++- .../create/CreateIndexRequestBuilder.java | 14 +++++ .../action/index/IndexRequest.java | 10 ++-- .../action/index/IndexRequestBuilder.java | 5 +- .../action/update/UpdateRequest.java | 9 +--- .../action/update/UpdateRequestBuilder.java | 7 +-- .../common/xcontent/XContentBuilder.java | 21 ++++++++ .../common/xcontent/XContentGenerator.java | 2 +- .../index/query/MoreLikeThisQueryBuilder.java | 2 +- .../elasticsearch/rest/RestController.java | 53 +++++++++---------- .../org/elasticsearch/rest/RestHandler.java | 4 ++ .../org/elasticsearch/rest/RestRequest.java | 4 +- .../create/CreateIndexRequestTests.java | 6 +-- .../rest/RestControllerTests.java | 15 +++++- 14 files changed, 103 insertions(+), 64 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 34cc694cec613..798c08bd180d0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -220,13 +220,24 @@ public CreateIndexRequest settings(Map source) { * * @param type The mapping type * @param source The mapping source - * @deprecated use {@link #mapping(String, BytesReference, XContentType)} to avoid content type detection + * @deprecated use {@link #mapping(String, String, XContentType)} to avoid content type detection */ @Deprecated public CreateIndexRequest mapping(String type, String source) { return mapping(type, new BytesArray(source), XContentFactory.xContentType(source)); } + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequest mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); + } + /** * Adds mapping that will be added when the index gets created. * @@ -234,7 +245,7 @@ public CreateIndexRequest mapping(String type, String source) { * @param source The mapping source * @param xContentType the content type of the mapping source */ - public CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) { + private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) { if (mappings.containsKey(type)) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 9642311ed59ea..237c88244b4cd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -115,12 +115,26 @@ public CreateIndexRequestBuilder setSettings(Map source) { * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} to avoid content type auto-detection */ + @Deprecated public CreateIndexRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + /** * The cause for this index creation. */ diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index b6ff97c693415..c75c37b3dabfa 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -335,15 +335,13 @@ public IndexRequest source(XContentBuilder sourceBuilder) { } /** - * Sets the content source to index. + * Sets the content source to index using the default content type ({@link Requests#INDEX_CONTENT_TYPE}) *

* Note: the number of objects passed to this method must be an even * number. Also the first argument in each pair (the field name) must have a * valid String representation. *

- * @deprecated use {@link #source(XContentType, Object...)} to be specific about your desired content type */ - @Deprecated public IndexRequest source(Object... source) { return source(Requests.INDEX_CONTENT_TYPE, source); } @@ -567,9 +565,7 @@ public void readFrom(StreamInput in) throws IOException { isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting - if (in.readBoolean()) { - contentType = XContentType.readFrom(in); - } + contentType = in.readOptionalWriteable(XContentType::readFrom); } else { contentType = XContentFactory.xContentType(source); } @@ -604,7 +600,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting if (contentType != null) { out.writeBoolean(true); - contentType.writeTo(out); + out.writeOptionalWriteable(contentType); } else { out.writeBoolean(false); } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index cd5a0c8ffaeee..7af43ec35eccb 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -205,11 +204,9 @@ public IndexRequestBuilder setSource(byte[] source, int offset, int length, XCon * number. Also the first argument in each pair (the field name) must have a * valid String representation. *

- * @deprecated use {@link #setSource(XContentType, Object...)} instead to be explicit about the content type */ - @Deprecated public IndexRequestBuilder setSource(Object... source) { - request.source(Requests.INDEX_CONTENT_TYPE, source); + request.source(source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 0cba05e721a72..504a297627d63 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -611,11 +610,9 @@ public UpdateRequest doc(byte[] source, int offset, int length, XContentType xCo /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. - * @deprecated use {@link #doc(XContentType, Object...)} to be explicit about content type */ - @Deprecated public UpdateRequest doc(Object... source) { - safeDoc().source(Requests.INDEX_CONTENT_TYPE, source); + safeDoc().source(source); return this; } @@ -729,11 +726,9 @@ public UpdateRequest upsert(byte[] source, int offset, int length, XContentType /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. - * @deprecated use {@link #upsert(XContentType, Object...)} to be explicit about content type */ - @Deprecated public UpdateRequest upsert(Object... source) { - safeUpsertRequest().source(Requests.INDEX_CONTENT_TYPE, source); + safeUpsertRequest().source(source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index f1be0a05e6a9e..1a4d4077b1045 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; @@ -279,9 +278,7 @@ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XConte /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. - * @deprecated use {@link #setDoc(XContentType, Object...)} to be specific about content type */ - @Deprecated public UpdateRequestBuilder setDoc(Object... source) { request.doc(source); return this; @@ -386,11 +383,9 @@ public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XCo /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. - * @deprecated use {@link #setUpsert(XContentType, Object...)} to be specific about the desired content type */ - @Deprecated public UpdateRequestBuilder setUpsert(Object... source) { - request.upsert(Requests.INDEX_CONTENT_TYPE, source); + request.upsert(source); return this; } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index a64310375107e..189e9d3c8d5dc 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -964,34 +964,55 @@ public XContentBuilder byteSizeField(String rawFieldName, String readableFieldNa // Raw fields ////////////////////////////////// + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #rawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ @Deprecated public XContentBuilder rawField(String name, InputStream value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the value taken from the bytes in the stream + */ public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException { generator.writeRawField(name, value, contentType); return this; } + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #rawField(String name, BytesReference, XContentType)} to avoid content type auto-detection + */ @Deprecated public XContentBuilder rawField(String name, BytesReference value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the given bytes as the value + */ public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException { generator.writeRawField(name, value, contentType); return this; } + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #rawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ @Deprecated public XContentBuilder rawValue(BytesReference value) throws IOException { generator.writeRawValue(value); return this; } + /** + * Writes a value with the source coming directly from the bytes + */ public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException { generator.writeRawValue(value, contentType); return this; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 9256555c14b86..60a188ca6ce58 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -100,7 +100,7 @@ public interface XContentGenerator extends Closeable, Flushable { /** * Writes a raw field with the given bytes as the value - * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType)} to avoid content type auto-detection */ @Deprecated void writeRawField(String name, BytesReference value) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index 3bbdfd798ad7c..ee0eff2a45162 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -337,7 +337,7 @@ public Item versionType(VersionType versionType) { return this; } - public XContentType xContentType() { + XContentType xContentType() { return xContentType; } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index c5724188f5ad0..fd3587df2aec2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -177,7 +177,9 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont assert contentLength >= 0 : "content length was negative, how is that possible?"; final RestHandler handler = getHandler(request); - if (contentLength == 0 || checkForContentType(request, handler)) { + if (contentLength > 0 && hasContentTypeOrCanAutoDetect(request, handler) == false) { + sendContentTypeErrorMessage(request, responseChannel); + } else { if (canTripCircuitBreaker(request)) { inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); } else { @@ -186,8 +188,6 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont // iff we could reserve bytes for the request we need to send the response also over this channel responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); dispatchRequest(request, responseChannel, client, threadContext, handler); - } else { - sendContentTypeErrorMessage(request, responseChannel); } } catch (Exception e) { try { @@ -204,26 +204,28 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final final RestHandler handler) throws Exception { if (checkRequestParameters(request, channel) == false) { channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled.")); - } - try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { - for (String key : headersToCopy) { - String httpHeader = request.header(key); - if (httpHeader != null) { - threadContext.putHeader(key, httpHeader); + } else { + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { + for (String key : headersToCopy) { + String httpHeader = request.header(key); + if (httpHeader != null) { + threadContext.putHeader(key, httpHeader); + } } - } - if (handler == null) { - if (request.method() == RestRequest.Method.OPTIONS) { - // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + if (handler == null) { + if (request.method() == RestRequest.Method.OPTIONS) { + // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) + + channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + } else { + final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; + channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + } } else { - final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; - channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); + wrappedHandler.handleRequest(request, channel, client); } - } else { - final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); - wrappedHandler.handleRequest(request, channel, client); } } } @@ -233,7 +235,7 @@ void dispatchRequest(final RestRequest request, final RestChannel channel, final * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text */ - boolean checkForContentType(final RestRequest restRequest, final RestHandler restHandler) { + private boolean hasContentTypeOrCanAutoDetect(final RestRequest restRequest, final RestHandler restHandler) { if (restRequest.getXContentType() == null) { if (restHandler != null && restHandler.supportsPlainText()) { // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can @@ -243,18 +245,11 @@ boolean checkForContentType(final RestRequest restRequest, final RestHandler res } else if (isContentTypeRequired) { return false; } else { - deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using" + + deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using " + "the [Content-Type] header."); XContentType xContentType = XContentFactory.xContentType(restRequest.content()); if (xContentType == null) { - if (restHandler != null) { - if (restHandler.supportsPlainText()) { - deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + - "in a supported format."); - } else { - return false; - } - } + return false; } else { restRequest.setXContentType(xContentType); } diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index b061e9ce763fe..740b98ae737e2 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -38,6 +38,10 @@ default boolean canTripCircuitBreaker() { return true; } + /** + * Indicates if a RestHandler supports plain text bodies + * @deprecated use request parameters or bodies that can be parsed with XContent! + */ @Deprecated default boolean supportsPlainText() { return false; diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index ed2929ca5bf83..7d41faf12e074 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -339,7 +339,7 @@ public final XContentParser contentParser() throws IOException { if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); } else if (xContentType.get() == null) { - throw new IllegalStateException("no content-type has been set so we cannot create a parser"); + throw new IllegalStateException("unknown content type"); } return xContentType.get().xContent().createParser(xContentRegistry, content); } @@ -402,7 +402,7 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { if (hasContent()) { if (xContentType.get() == null) { - throw new IllegalStateException("Content-Type must be provided"); + throw new IllegalStateException("unknown content type"); } return new Tuple<>(xContentType.get(), content()); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java index c9ce52a5d8a0c..590eba3666610 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -34,8 +34,8 @@ public class CreateIndexRequestTests extends ESTestCase { public void testSerialization() throws IOException { CreateIndexRequest request = new CreateIndexRequest("foo"); - BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes(); - request.mapping("my_type", bytesReference, XContentType.JSON); + String mapping = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().string(); + request.mapping("my_type", mapping, XContentType.JSON); try (BytesStreamOutput output = new BytesStreamOutput()) { request.writeTo(output); @@ -44,7 +44,7 @@ public void testSerialization() throws IOException { CreateIndexRequest serialized = new CreateIndexRequest(); serialized.readFrom(in); assertEquals(request.index(), serialized.index()); - assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type")); + assertEquals(mapping, serialized.mappings().get("my_type")); } } } diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 26be37c1ea1a6..79c6f6561e6fa 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.rest; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -277,7 +276,7 @@ public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() { public void testDispatchWorksWithPlainText() { String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt()); FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) - .withContent(new BytesArray(content.getBytes(StandardCharsets.UTF_8)), null).withPath("/foo") + .withContent(new BytesArray(content), null).withPath("/foo") .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build(); AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { @@ -298,6 +297,18 @@ public boolean supportsPlainText() { assertWarnings("Plain text request bodies are deprecated. Use request parameters or body in a supported format."); } + public void testDispatchWorksWithAutoDetection() { + FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withContent(new BytesArray("{}"), null).withPath("/") + .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("application/x-www-form-urlencoded"))).build(); + AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK); + + assertFalse(channel.sendResponseCalled.get()); + restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY)); + assertTrue(channel.sendResponseCalled.get()); + assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header."); + } + private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { From 7b89522cd62113c069a0c1ded9391f03c28e7074 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 30 Jan 2017 10:22:15 -0500 Subject: [PATCH 28/34] more feedback changes --- .../action/index/IndexRequest.java | 1 - .../common/xcontent/XContentHelper.java | 44 ++++++++++++++++++- .../index/mapper/DocumentParser.java | 1 + .../index/mapper/ParsedDocument.java | 12 ++++- .../index/IndexingSlowLogTests.java | 3 +- .../index/engine/InternalEngineTests.java | 4 +- .../index/engine/ShadowEngineTests.java | 4 +- .../index/shard/IndexShardIT.java | 7 +-- .../index/shard/IndexShardTests.java | 4 +- .../index/shard/RefreshListenersTests.java | 4 +- .../index/translog/TranslogTests.java | 4 +- .../PercolateQueryBuilderTests.java | 2 +- .../reindex/remote/RemoteResponseParsers.java | 39 +++++----------- .../remote/RemoteScrollableHitSource.java | 5 +-- 14 files changed, 90 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index c75c37b3dabfa..dafbbfb2ff59b 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -599,7 +599,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(autoGeneratedTimestamp); if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting if (contentType != null) { - out.writeBoolean(true); out.writeOptionalWriteable(contentType); } else { out.writeBoolean(false); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 9fc71b6838cd7..dd7508280d9ca 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -41,6 +41,11 @@ @SuppressWarnings("unchecked") public class XContentHelper { + /** + * Creates a parser based on the bytes provided + * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException { Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -48,13 +53,31 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - XContentType contentType = XContentFactory.xContentType(compressedInput); + final XContentType contentType = XContentFactory.xContentType(compressedInput); return XContentFactory.xContent(contentType).createParser(xContentRegistry, compressedInput); } else { return XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes.streamInput()); } } + /** + * Creates a parser for the bytes using the supplied content-type + */ + public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes, + XContentType xContentType) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + InputStream compressedInput = compressor.streamInput(bytes.streamInput()); + if (compressedInput.markSupported() == false) { + compressedInput = new BufferedInputStream(compressedInput); + } + return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput); + } else { + return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput()); + } + } + /** * Converts the given bytes into a map that is optionally ordered. * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)} @@ -390,7 +413,10 @@ public static void copyCurrentEvent(XContentGenerator generator, XContentParser /** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}. + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type + * auto-detection */ + @Deprecated public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { @@ -401,6 +427,22 @@ public static void writeRawField(String field, BytesReference source, XContentBu } } + /** + * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using + * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference, XContentType)}. + */ + public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder, + ToXContent.Params params) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(source); + if (compressor != null) { + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + builder.rawField(field, compressedStreamInput, xContentType); + } else { + builder.rawField(field, source, xContentType); + } + } + /** * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided * {@link XContentType}. Wraps the output into a new anonymous object. diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index a35b06a06ad77..be69c40dab941 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -153,6 +153,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext. source.routing(), context.docs(), context.sourceToParse().source(), + context.sourceToParse().getXContentType(), update ).parent(source.parent()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index db8bdf9df7f6c..f7d5804be0d53 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -22,8 +22,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.SeqNoFieldMapper; import java.util.List; @@ -43,6 +43,7 @@ public class ParsedDocument { private final List documents; private BytesReference source; + private XContentType xContentType; private Mapping dynamicMappingsUpdate; @@ -55,6 +56,7 @@ public ParsedDocument(Field version, String routing, List documents, BytesReference source, + XContentType xContentType, Mapping dynamicMappingsUpdate) { this.version = version; this.seqID = seqID; @@ -65,6 +67,7 @@ public ParsedDocument(Field version, this.documents = documents; this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; + this.xContentType = xContentType; } public BytesRef uid() { @@ -105,8 +108,13 @@ public BytesReference source() { return this.source; } - public void setSource(BytesReference source) { + public XContentType getXContentType() { + return this.xContentType; + } + + public void setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; } public ParsedDocument parent(String parent) { diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index ecdba6ec44f1e..c815b2b55f9f3 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexingSlowLog.SlowLogParsedDocumentPrinter; import org.elasticsearch.index.mapper.ParsedDocument; @@ -41,7 +42,7 @@ public class IndexingSlowLogTests extends ESTestCase { public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes(); ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), SeqNoFieldMapper.SequenceID.emptySeqID(), "id", - "test", null, null, source, null); + "test", null, null, source, XContentType.JSON, null); Index index = new Index("foo", "123"); // Turning off document logging doesn't log source[] SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 0); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 9b17341c1554a..94b95d1aabf61 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -88,6 +88,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -290,7 +291,8 @@ private static ParsedDocument testParsedDocument(String id, String type, String document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingUpdate); } protected Store createStore() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index 219af4af4daa9..53708b28dfb0f 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.Mapping; @@ -179,7 +180,8 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingsUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingsUpdate); } protected Store createStore(Path p) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 7af2affc8150c..97c96c8af12f7 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -103,7 +103,8 @@ protected Collection> getPlugins() { } private ParsedDocument testParsedDocument(String id, String type, String routing, long seqNo, - ParseContext.Document document, BytesReference source, Mapping mappingUpdate) { + ParseContext.Document document, BytesReference source, XContentType xContentType, + Mapping mappingUpdate) { Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); @@ -113,7 +114,7 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); return new ParsedDocument(versionField, seqID, id, type, routing, - Collections.singletonList(document), source, mappingUpdate); + Collections.singletonList(document), source, xContentType, mappingUpdate); } public void testLockTryingToDelete() throws Exception { @@ -333,7 +334,7 @@ public void testMaybeFlush() throws Exception { null, SequenceNumbersService.UNASSIGNED_SEQ_NO, new ParseContext.Document(), - new BytesArray(new byte[]{1}), null); + new BytesArray(new byte[]{1}), XContentType.JSON, null); Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); shard.index(index); assertTrue(shard.shouldFlush()); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 79e3868da46e7..4a03a2632322d 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -558,7 +559,8 @@ private ParsedDocument testParsedDocument(String id, String type, String routing document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate); + return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON, + mappingUpdate); } public void testIndexingOperationsListeners() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index d5c10dddc3653..846ca6be2014a 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; @@ -338,7 +339,8 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); BytesReference source = new BytesArray(new byte[] { 1 }); - ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, null); + ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, XContentType.JSON, + null); Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); return engine.index(index); } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 1a9d7c97dc355..0591500423902 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; @@ -2028,7 +2029,8 @@ public void testTranslogOpSerialization() throws Exception { document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); document.add(seqID.primaryTerm); - ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, null); + ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, XContentType.JSON, + null); Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm, 1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index 6f1e80d80b8f7..23f3fc77c2055 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -234,7 +234,7 @@ public void testCreateMultiDocumentSearcher() throws Exception { } Analyzer analyzer = new WhitespaceAnalyzer(); - ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null); + ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null, null); IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs)); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java index 634634fd9e6ab..246780132d858 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; import org.elasticsearch.index.reindex.ScrollableHitSource.Response; @@ -39,7 +38,6 @@ import java.io.IOException; import java.util.List; -import java.util.Objects; import java.util.function.BiFunction; import static java.util.Collections.emptyList; @@ -57,7 +55,7 @@ private RemoteResponseParsers() {} /** * Parser for an individual {@code hit} element. */ - public static final ConstructingObjectParser HIT_PARSER = + public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { int i = 0; String index = (String) a[i++]; @@ -77,10 +75,10 @@ private RemoteResponseParsers() {} * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for * now this should do. */ - try (XContentBuilder b = XContentBuilder.builder(s.getXContentType().xContent())) { + try (XContentBuilder b = XContentBuilder.builder(s.xContent())) { b.copyCurrentStructure(p); // a hack but this lets us get the right xcontent type to go with the source - return new Tuple<>(b.bytes(), s.getXContentType()); + return new Tuple<>(b.bytes(), s); } } catch (IOException e) { throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e); @@ -96,7 +94,7 @@ class Fields { String routing; String parent; } - ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); + ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); hit.setParent(fields.parent); @@ -109,7 +107,7 @@ class Fields { /** * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}. */ - public static final ConstructingObjectParser HITS_PARSER = + public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); static { HITS_PARSER.declareLong(constructorArg(), new ParseField("total")); @@ -119,7 +117,7 @@ class Fields { /** * Parser for {@code failed} shards in the {@code _shards} elements. */ - public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = + public static final ConstructingObjectParser SEARCH_FAILURE_PARSER = new ConstructingObjectParser<>("failure", true, a -> { int i = 0; String index = (String) a[i++]; @@ -152,7 +150,7 @@ class Fields { * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it * parses to an empty list. */ - public static final ConstructingObjectParser, ResponseContext> SHARDS_PARSER = + public static final ConstructingObjectParser, XContentType> SHARDS_PARSER = new ConstructingObjectParser<>("_shards", true, a -> { @SuppressWarnings("unchecked") List failures = (List) a[0]; @@ -163,7 +161,7 @@ class Fields { SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures")); } - public static final ConstructingObjectParser RESPONSE_PARSER = + public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>("search_response", true, a -> { int i = 0; Throwable catastrophicFailure = (Throwable) a[i++]; @@ -202,9 +200,9 @@ class Fields { * Collects stuff about Throwables and attempts to rebuild them. */ public static class ThrowableBuilder { - public static final BiFunction PARSER; + public static final BiFunction PARSER; static { - ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); + ObjectParser parser = new ObjectParser<>("reason", true, ThrowableBuilder::new); PARSER = parser.andThen(ThrowableBuilder::build); parser.declareString(ThrowableBuilder::setType, new ParseField("type")); parser.declareString(ThrowableBuilder::setReason, new ParseField("reason")); @@ -268,25 +266,12 @@ public void setCausedBy(Throwable causedBy) { /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ - public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( "/", true, a -> (Version) a[0]); static { - ConstructingObjectParser versionParser = new ConstructingObjectParser<>( + ConstructingObjectParser versionParser = new ConstructingObjectParser<>( "version", true, a -> Version.fromString((String) a[0])); versionParser.declareString(constructorArg(), new ParseField("number")); MAIN_ACTION_PARSER.declareObject(constructorArg(), versionParser, new ParseField("version")); } - - static final class ResponseContext { - - private final XContentType xContentType; - - ResponseContext(XContentType xContentType) { - this.xContentType = Objects.requireNonNull(xContentType); - } - - XContentType getXContentType() { - return this.xContentType; - } - } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java index 0368a8b3797d2..43aa8ade4e987 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -45,7 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.reindex.ScrollableHitSource; -import org.elasticsearch.index.reindex.remote.RemoteResponseParsers.ResponseContext; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -143,7 +142,7 @@ protected void cleanup() { } private void execute(String method, String uri, Map params, HttpEntity entity, - BiFunction parser, Consumer listener) { + BiFunction parser, Consumer listener) { // Preserve the thread context so headers survive after the call java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); class RetryHelper extends AbstractRunnable { @@ -179,7 +178,7 @@ public void onSuccess(org.elasticsearch.client.Response response) { // EMPTY is safe here because we don't call namedObject try (XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) { - parsedResponse = parser.apply(xContentParser, new ResponseContext(xContentType)); + parsedResponse = parser.apply(xContentParser, xContentType); } catch (ParsingException e) { /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it * is totally wrong and we're probably not talking to Elasticsearch. */ From 5270a4482e14460d16d41ad72498267e338994df Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 30 Jan 2017 11:00:00 -0500 Subject: [PATCH 29/34] add comment --- .../elasticsearch/test/rest/yaml/ClientYamlTestResponse.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java index d1027156c74b7..f47cfd68d4e19 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java @@ -87,7 +87,9 @@ public List getWarningHeaders() { && response.getRequestLine().getUri().contains("source") && response.getRequestLine().getUri().contains("source_content_type") == false && header.getValue().startsWith( "Deprecated use of the [source] parameter without the [source_content_type] parameter.")) { - // this is because we do not sent the source content type header when the node is 5.3.0 or below + // this is because we do not send the source content type header when the node is 5.3.0 or below and the request + // might have been sent to a node with a version > 5.3.0 when running backwards 5.0 tests. The Java RestClient + // has control of the node the request is sent to so we can only detect this after the fact right now // TODO remove this when we bump versions } else { warningHeaders.add(header.getValue()); From e4f46d38803010e5144a7252cc9af4960f28cc79 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 30 Jan 2017 11:37:21 -0500 Subject: [PATCH 30/34] use writeOptionalWriteable the right way --- .../java/org/elasticsearch/action/index/IndexRequest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index dafbbfb2ff59b..7e3f14368debe 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -598,11 +598,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting - if (contentType != null) { - out.writeOptionalWriteable(contentType); - } else { - out.writeBoolean(false); - } + out.writeOptionalWriteable(contentType); } } From 4bd706aea81c80db2e1d0b03dac8bf7c067b7018 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 30 Jan 2017 11:55:01 -0500 Subject: [PATCH 31/34] use xcontent type to avoid deprecated method --- core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 8672bbc8351ea..b1d8ac188626f 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -190,7 +190,7 @@ public String toString() { return sb.toString(); } try { - String source = XContentHelper.convertToJson(doc.source(), reformat); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]"); } catch (IOException e) { sb.append(", source[_failed_to_convert_]"); From dd19f71913c625685dad7841b81daafbfae517e3 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 1 Feb 2017 09:35:05 -0500 Subject: [PATCH 32/34] add missing setting registration --- .../java/org/elasticsearch/common/settings/ClusterSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 279be58e01d4d..69adc5878ff01 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -237,6 +237,7 @@ public void apply(Settings value, Settings current, Settings previous) { HttpTransportSettings.SETTING_CORS_ALLOW_METHODS, HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS, HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED, + HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH, HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE, HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE, From fd62a34a34b6101ed13207dc2ab378ca2414a736 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 1 Feb 2017 18:29:41 -0500 Subject: [PATCH 33/34] use 5.3.0-snapshot for rolling upgrade --- qa/rolling-upgrade/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index b03d12fbdf787..0d80c6ffd19d9 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -25,7 +25,7 @@ task oldClusterTest(type: RestIntegTestTask) { mustRunAfter(precommit) cluster { distribution = 'zip' - bwcVersion = '5.2.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop + bwcVersion = '5.3.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop numBwcNodes = 2 numNodes = 2 clusterName = 'rolling-upgrade' From 159b45094cc237349788f54ccb0c06233f793223 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 2 Feb 2017 13:13:12 -0500 Subject: [PATCH 34/34] add status --- core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java index d7d837600c613..a7b709625ce59 100644 --- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java +++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java @@ -150,6 +150,7 @@ private static XContentBuilder build(RestChannel channel, RestStatus status, Exc static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException { return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject() .field("error", errorMessage) + .field("status", status.getStatus()) .endObject()); }