diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java index 4815353936b28..53daf463941ff 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java @@ -40,6 +40,7 @@ import static org.elasticsearch.client.RequestConverters.createEntity; import static org.elasticsearch.client.transform.DeleteTransformRequest.FORCE; import static org.elasticsearch.client.transform.GetTransformRequest.ALLOW_NO_MATCH; +import static org.elasticsearch.client.transform.GetTransformRequest.EXCLUDE_GENERATED; import static org.elasticsearch.client.transform.PutTransformRequest.DEFER_VALIDATION; import static org.elasticsearch.client.transform.StopTransformRequest.WAIT_FOR_CHECKPOINT; @@ -89,6 +90,9 @@ static Request getTransform(GetTransformRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { request.addParameter(ALLOW_NO_MATCH, getRequest.getAllowNoMatch().toString()); } + if (getRequest.getExcludeGenerated() != null) { + request.addParameter(EXCLUDE_GENERATED, getRequest.getExcludeGenerated().toString()); + } return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java index f0238083f6af0..184222244732b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java @@ -30,6 +30,7 @@ public class GetTransformRequest implements Validatable { + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String ALLOW_NO_MATCH = "allow_no_match"; /** * Helper method to create a request that will get ALL Transforms @@ -42,6 +43,7 @@ public static GetTransformRequest getAllTransformRequest() { private final List ids; private PageParams pageParams; private Boolean allowNoMatch; + private Boolean excludeGenerated; public GetTransformRequest(String... ids) { this.ids = Arrays.asList(ids); @@ -67,6 +69,14 @@ public void setAllowNoMatch(Boolean allowNoMatch) { this.allowNoMatch = allowNoMatch; } + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; + } + + public Boolean getExcludeGenerated() { + return excludeGenerated; + } + @Override public Optional validate() { if (ids == null || ids.isEmpty()) { @@ -80,7 +90,7 @@ public Optional validate() { @Override public int hashCode() { - return Objects.hash(ids, pageParams, allowNoMatch); + return Objects.hash(ids, pageParams, excludeGenerated, allowNoMatch); } @Override @@ -95,6 +105,7 @@ public boolean equals(Object obj) { GetTransformRequest other = (GetTransformRequest) obj; return Objects.equals(ids, other.ids) && Objects.equals(pageParams, other.pageParams) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java index c9c7d9d806b3f..766f3f2a292e2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java @@ -687,6 +687,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException // tag::get-transform-request-options request.setPageParams(new PageParams(0, 100)); // <1> request.setAllowNoMatch(true); // <2> + request.setExcludeGenerated(false); // <3> // end::get-transform-request-options // tag::get-transform-execute diff --git a/docs/java-rest/high-level/transform/get_transform.asciidoc b/docs/java-rest/high-level/transform/get_transform.asciidoc index 64aa0f229c427..ca79ebf17e1ba 100644 --- a/docs/java-rest/high-level/transform/get_transform.asciidoc +++ b/docs/java-rest/high-level/transform/get_transform.asciidoc @@ -34,6 +34,9 @@ include-tagged::{doc-tests-file}[{api}-request-options] {transforms} to skip. `size` specifies the maximum number of {transforms} to get. Defaults to `0` and `100` respectively. <2> Whether to ignore if a wildcard expression matches no {transforms}. +<3> Optional boolean value for requesting the {transform} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {transform} is created are removed. include::../execution.asciidoc[] diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index 939bc59413d4a..c9a80ab113569 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -26,12 +26,12 @@ Retrieves configuration information for {transforms}. [[get-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: -* `monitor_transform` +* `monitor_transform` -The built-in `transform_user` role has this privilege. +The built-in `transform_user` role has this privilege. For more information, see <> and <>. @@ -49,7 +49,7 @@ specifying `*` as the ``, or by omitting the ``. ``:: (Optional, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] - + [[get-transform-query-parms]] == {api-query-parms-title} @@ -65,6 +65,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] +`exclude_generated`:: +(Optional, boolean) +Excludes fields that were automatically added when creating the transform. +This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. + [[get-transform-response]] == {api-response-body-title} @@ -79,13 +85,13 @@ This property is informational; you cannot change its value. `version`:: (string) The version of {es} that existed on the node when the {transform} was created. - + [[get-transform-response-codes]] == {api-response-codes-title} `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no - resources that match the request or only partial matches for the request. + resources that match the request or only partial matches for the request. [[get-transform-example]] == {api-examples-title} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 46855b74ab210..cb52699c5b26d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -84,6 +84,7 @@ public final class TransformField { * API's) */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; // internal document id public static String DOCUMENT_ID_FIELD = "_id"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java index b3478e89520c0..3e30482daef0c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.license.RemoteClusterLicenseChecker; +import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; import java.io.IOException; @@ -113,7 +114,11 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.array(INDEX.getPreferredName(), index); - builder.field(QUERY.getPreferredName(), queryConfig); + if (params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false) == false) { + builder.field(QUERY.getPreferredName(), queryConfig); + } else if(queryConfig.equals(QueryConfig.matchAll()) == false) { + builder.field(QUERY.getPreferredName(), queryConfig); + } builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 2f1995fd0afe0..d5b3e7ddb74a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -342,9 +342,31 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); + final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); + assert (forInternalStorage && excludeGenerated) == false: + "unsupported behavior, exclude_generated is true and for_internal_storage is true"; builder.startObject(); builder.field(TransformField.ID.getPreferredName(), id); - builder.field(TransformField.SOURCE.getPreferredName(), source); + if (excludeGenerated == false) { + if (headers.isEmpty() == false && forInternalStorage) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (transformVersion != null) { + builder.field(TransformField.VERSION.getPreferredName(), transformVersion); + } + if (createTime != null) { + builder.timeField( + TransformField.CREATE_TIME.getPreferredName(), + TransformField.CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); + } + if (forInternalStorage) { + builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); + } + } + builder.field(TransformField.SOURCE.getPreferredName(), source, params); builder.field(TransformField.DESTINATION.getPreferredName(), dest); if (frequency != null) { builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep()); @@ -357,26 +379,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (pivotConfig != null) { builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); } - if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); - } - if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (description != null) { builder.field(TransformField.DESCRIPTION.getPreferredName(), description); } builder.field(TransformField.SETTINGS.getPreferredName(), settings); - if (transformVersion != null) { - builder.field(TransformField.VERSION.getPreferredName(), transformVersion); - } - if (createTime != null) { - builder.timeField( - TransformField.CREATE_TIME.getPreferredName(), - TransformField.CREATE_TIME.getPreferredName() + "_string", - createTime.toEpochMilli() - ); - } builder.endObject(); return builder; } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json index 17f333861c5dc..22726a9596332 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json @@ -50,6 +50,12 @@ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)" + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits generated fields. Allows transform configurations to be easily copied between clusters and within the same cluster" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json index 1ab97aa6a18db..69e87d9755477 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json @@ -42,6 +42,12 @@ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)" + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on transform PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml index 20b49c361b18f..5fd1a726602c5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml @@ -670,3 +670,34 @@ setup: }, "description": "yaml test transform on airline-data" } + +--- +"Test transform for export": + - do: + transform.put_transform: + transform_id: "airline-transform" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "description": "yaml test transform on airline-data" + } + - match: { acknowledged: true } + + - do: + transform.get_transform: + transform_id: "airline-transform" + exclude_generated: true + + - match: {transforms.0.source.index: ["airline-data"]} + - match: {transforms.0.dest.index: "airline-data-by-airline"} + - match: {transforms.0.pivot.group_by.airline.terms.field: "airline"} + - match: {transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime"} + - match: {transforms.0.description: "yaml test transform on airline-data"} + - match: {transforms.0.id: "airline-transform"} + - is_false: transforms.0.create_time + - is_false: transforms.0.version diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index e100fc6625743..b4dd50b0983fe 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -9,6 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -1686,6 +1687,38 @@ public void testPivotWithFilter() throws Exception { assertEquals(3, actual.longValue()); } + @SuppressWarnings("unchecked") + public void testExportAndImport() throws Exception { + String transformId = "export-transform"; + String transformIndex = "export_reviews"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex); + + createPivotReviewsTransform(transformId, transformIndex, null, null, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS); + + Response response = adminClient().performRequest(new Request("GET", + getTransformEndpoint() + transformId + "?exclude_generated=true")); + Map storedConfig = ((List>) XContentMapValues.extractValue( + "transforms", + entityAsMap(response))) + .get(0); + storedConfig.remove("id"); + try (XContentBuilder builder = jsonBuilder()) { + builder.map(storedConfig); + Request putTransform = new Request("PUT", getTransformEndpoint() + transformId + "-import"); + putTransform.setJsonEntity(Strings.toString(builder)); + adminClient().performRequest(putTransform); + } + + response = adminClient().performRequest(new Request("GET", + getTransformEndpoint() + transformId + "-import" + "?exclude_generated=true")); + Map importConfig = ((List>) XContentMapValues.extractValue( + "transforms", + entityAsMap(response))) + .get(0); + importConfig.remove("id"); + assertThat(storedConfig, equalTo(importConfig)); + } + private void createDateNanoIndex(String indexName, int numDocs) throws IOException { // create mapping try (XContentBuilder builder = jsonBuilder()) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java index f57e830debe7b..a6b599aafde36 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java @@ -14,10 +14,13 @@ import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.GetTransformAction; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED; public class RestGetTransformAction extends BaseRestHandler { @@ -47,4 +50,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public String getName() { return "transform_get_transform_action"; } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java index 211e197a4b736..dcf0a49416731 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java @@ -18,9 +18,11 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED; public class RestGetTransformActionDeprecated extends BaseRestHandler { @Override @@ -56,4 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public String getName() { return "data_frame_get_transforms_action"; } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } }