From 1c0a59ea28b18fc288627b3747b8566fb2b012cb Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 20 Jul 2018 14:04:07 +0200 Subject: [PATCH] INGEST: Add Configuration Except. Data to Metdata * closes #27728 --- .../common/AppendProcessorFactoryTests.java | 2 +- .../common/ConvertProcessorFactoryTests.java | 6 +++--- .../common/FailProcessorFactoryTests.java | 2 +- .../common/RemoveProcessorFactoryTests.java | 2 +- .../common/SetProcessorFactoryTests.java | 2 +- .../rest-api-spec/test/ingest/20_crud.yml | 6 +++--- .../test/ingest/50_on_failure.yml | 6 +++--- .../rest-api-spec/test/ingest/90_simulate.yml | 18 +++++++++--------- .../10_pipeline_with_mustache_templates.yml | 2 +- .../50_script_processor_using_painless.yml | 2 +- .../ingest/ConfigurationUtils.java | 14 +++++++------- .../ingest/ConfigurationUtilsTests.java | 18 +++++++++--------- .../ingest/PipelineStoreTests.java | 4 ++-- .../SetSecurityUserProcessorFactoryTests.java | 12 ++++++------ 14 files changed, 48 insertions(+), 48 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index 39a7bfd9a20b2..d51cb368e4317 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -100,6 +100,6 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); - assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag)); + assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java index 9e4acd7b17f83..f3396da64eb5f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java @@ -58,9 +58,9 @@ public void testCreateUnsupportedType() throws Exception { fail("factory create should have failed"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field.")); - assertThat(e.getHeader("processor_type").get(0), equalTo(ConvertProcessor.TYPE)); - assertThat(e.getHeader("property_name").get(0), equalTo("type")); - assertThat(e.getHeader("processor_tag"), nullValue()); + assertThat(e.getMetadata("es.processor_type").get(0), equalTo(ConvertProcessor.TYPE)); + assertThat(e.getMetadata("es.property_name").get(0), equalTo("type")); + assertThat(e.getMetadata("es.processor_tag"), nullValue()); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java index 801441407a7f7..3c89778f0e825 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java @@ -66,6 +66,6 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); - assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag)); + assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java index c439a9662f202..bebe780276208 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java @@ -79,6 +79,6 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); - assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag)); + assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java index 59a99b8f995d8..9602f34f698f7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java @@ -108,7 +108,7 @@ public void testInvalidMustacheTemplate() throws Exception { String processorTag = randomAlphaOfLength(10); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config)); assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); - assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag)); + assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml index 0e348bbd7265d..bd6a3e6ca14fd 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml @@ -158,9 +158,9 @@ teardown: } - match: { error.root_cause.0.type: "parse_exception" } - match: { error.root_cause.0.reason: "[field] required property is missing" } - - match: { error.root_cause.0.header.processor_tag: "fritag" } - - match: { error.root_cause.0.header.processor_type: "set" } - - match: { error.root_cause.0.header.property_name: "field" } + - match: { error.root_cause.0.processor_tag: "fritag" } + - match: { error.root_cause.0.processor_type: "set" } + - match: { error.root_cause.0.property_name: "field" } --- "Test basic pipeline with on_failure in processor": diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml index 4b40d9f670bfe..718b91ac1c111 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml @@ -148,9 +148,9 @@ teardown: } - match: { error.root_cause.0.type: "parse_exception" } - match: { error.root_cause.0.reason: "[on_failure] processors list cannot be empty" } - - match: { error.root_cause.0.header.processor_type: "fail" } - - match: { error.root_cause.0.header.processor_tag: "emptyfail" } - - match: { error.root_cause.0.header.property_name: "on_failure" } + - match: { error.root_cause.0.processor_type: "fail" } + - match: { error.root_cause.0.processor_tag: "emptyfail" } + - match: { error.root_cause.0.property_name: "on_failure" } --- "Test pipeline with empty on_failure in pipeline": diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml index 8b3ed313314bb..776a8af0c2420 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml @@ -107,9 +107,9 @@ teardown: } - match: { error.root_cause.0.type: "parse_exception" } - match: { error.root_cause.0.reason: "[field] required property is missing" } - - match: { error.root_cause.0.header.processor_tag: "fails" } - - match: { error.root_cause.0.header.processor_type: "set" } - - match: { error.root_cause.0.header.property_name: "field" } + - match: { error.root_cause.0.processor_tag: "fails" } + - match: { error.root_cause.0.processor_type: "set" } + - match: { error.root_cause.0.property_name: "field" } --- "Test simulate without index type and id": @@ -198,9 +198,9 @@ teardown: } ] } - - is_false: error.root_cause.0.header.processor_type - - is_false: error.root_cause.0.header.processor_tag - - match: { error.root_cause.0.header.property_name: "pipeline" } + - is_false: error.root_cause.0.processor_type + - is_false: error.root_cause.0.processor_tag + - match: { error.root_cause.0.property_name: "pipeline" } - match: { error.reason: "[pipeline] required property is missing" } --- @@ -233,9 +233,9 @@ teardown: } - match: { error.root_cause.0.type: "parse_exception" } - match: { error.root_cause.0.reason: "[value] required property is missing" } - - match: { error.root_cause.0.header.processor_type: "set" } - - match: { error.root_cause.0.header.property_name: "value" } - - is_false: error.root_cause.0.header.processor_tag + - match: { error.root_cause.0.processor_type: "set" } + - match: { error.root_cause.0.property_name: "value" } + - is_false: error.root_cause.0.processor_tag --- "Test simulate with verbose flag": diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml index 6bc6219bfe59d..5ffaebdcaef36 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml @@ -347,6 +347,6 @@ } ] } - - match: { error.header.processor_type: "set" } + - match: { error.processor_type: "set" } - match: { error.type: "script_exception" } - match: { error.reason: "Mustache function [join] must contain one and only one identifier" } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yml index 1c027adcc8071..cce388c2ff737 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yml @@ -103,7 +103,7 @@ } ] } - - match: { error.header.processor_type: "script" } + - match: { error.processor_type: "script" } - match: { error.type: "script_exception" } - match: { error.reason: "compile error" } diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index 78dc0ec6bfef1..219008b840ca5 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -284,14 +284,14 @@ public static ElasticsearchException newConfigurationException(String processorT msg = "[" + propertyName + "] " + reason; } ElasticsearchParseException exception = new ElasticsearchParseException(msg); - addHeadersToException(exception, processorType, processorTag, propertyName); + addMetadataToException(exception, processorType, processorTag, propertyName); return exception; } public static ElasticsearchException newConfigurationException(String processorType, String processorTag, String propertyName, Exception cause) { ElasticsearchException exception = ExceptionsHelper.convertToElastic(cause); - addHeadersToException(exception, processorType, processorTag, propertyName); + addMetadataToException(exception, processorType, processorTag, propertyName); return exception; } @@ -341,16 +341,16 @@ public String execute() { } } - private static void addHeadersToException(ElasticsearchException exception, String processorType, - String processorTag, String propertyName) { + private static void addMetadataToException(ElasticsearchException exception, String processorType, + String processorTag, String propertyName) { if (processorType != null) { - exception.addHeader("processor_type", processorType); + exception.addMetadata("es.processor_type", processorType); } if (processorTag != null) { - exception.addHeader("processor_tag", processorTag); + exception.addMetadata("es.processor_tag", processorTag); } if (propertyName != null) { - exception.addHeader("property_name", propertyName); + exception.addMetadata("es.property_name", propertyName); } } diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index af863410f9f35..61afd9ce2a473 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -131,9 +131,9 @@ public void testReadProcessors() throws Exception { ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> ConfigurationUtils.readProcessorConfigs(config, registry)); assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]")); - assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown"))); - assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor"))); - assertThat(e.getHeader("property_name"), is(nullValue())); + assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown"))); + assertThat(e.getMetadata("es.processor_type"), equalTo(Collections.singletonList("unknown_processor"))); + assertThat(e.getMetadata("es.property_name"), is(nullValue())); List> config2 = new ArrayList<>(); unknownTaggedConfig = new HashMap<>(); @@ -144,17 +144,17 @@ public void testReadProcessors() throws Exception { config2.add(Collections.singletonMap("second_unknown_processor", secondUnknonwTaggedConfig)); e = expectThrows(ElasticsearchParseException.class, () -> ConfigurationUtils.readProcessorConfigs(config2, registry)); assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]")); - assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown"))); - assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor"))); - assertThat(e.getHeader("property_name"), is(nullValue())); + assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown"))); + assertThat(e.getMetadata("es.processor_type"), equalTo(Collections.singletonList("unknown_processor"))); + assertThat(e.getMetadata("es.property_name"), is(nullValue())); assertThat(e.getSuppressed().length, equalTo(1)); assertThat(e.getSuppressed()[0], instanceOf(ElasticsearchParseException.class)); ElasticsearchParseException e2 = (ElasticsearchParseException) e.getSuppressed()[0]; assertThat(e2.getMessage(), equalTo("No processor type exists with name [second_unknown_processor]")); - assertThat(e2.getHeader("processor_tag"), equalTo(Collections.singletonList("my_second_unknown"))); - assertThat(e2.getHeader("processor_type"), equalTo(Collections.singletonList("second_unknown_processor"))); - assertThat(e2.getHeader("property_name"), is(nullValue())); + assertThat(e2.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_second_unknown"))); + assertThat(e2.getMetadata("es.processor_type"), equalTo(Collections.singletonList("second_unknown_processor"))); + assertThat(e2.getMetadata("es.property_name"), is(nullValue())); } public void testReadProcessorFromObjectOrMap() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java index 250bb5059cf58..d0ce465fc9ef8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java @@ -356,8 +356,8 @@ public void testValidate() throws Exception { ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> store.validatePipeline(ingestInfos, putRequest)); assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage()); - assertEquals("remove", e.getHeader("processor_type").get(0)); - assertEquals("tag2", e.getHeader("processor_tag").get(0)); + assertEquals("remove", e.getMetadata("es.processor_type").get(0)); + assertEquals("tag2", e.getMetadata("es.processor_tag").get(0)); ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove")))); store.validatePipeline(ingestInfos, putRequest); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java index 19da9b18ea650..01acbc19c2a04 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java @@ -31,9 +31,9 @@ public void testProcessor_noField() throws Exception { SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null); Map config = new HashMap<>(); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); - assertThat(e.getHeader("property_name").get(0), equalTo("field")); - assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); - assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag")); + assertThat(e.getMetadata("es.property_name").get(0), equalTo("field")); + assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); + assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); } public void testProcessor_validProperties() throws Exception { @@ -52,9 +52,9 @@ public void testProcessor_invalidProperties() throws Exception { config.put("field", "_field"); config.put("properties", Arrays.asList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config)); - assertThat(e.getHeader("property_name").get(0), equalTo("properties")); - assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); - assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag")); + assertThat(e.getMetadata("es.property_name").get(0), equalTo("properties")); + assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE)); + assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag")); } }