Skip to content

Commit

Permalink
Configurable output format for date processor (#61324) (#62175)
Browse files Browse the repository at this point in the history
  • Loading branch information
@danhermann
danhermann authored Sep 9, 2020
1 parent e181e24 commit eeeb355
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 3 deletions.
27 changes: 24 additions & 3 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,24 @@ public final class DateProcessor extends AbstractProcessor {

public static final String TYPE = "date";
static final String DEFAULT_TARGET_FIELD = "@timestamp";
private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
static final String DEFAULT_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";

private final DateFormatter formatter;
private final TemplateScript.Factory timezone;
private final TemplateScript.Factory locale;
private final String field;
private final String targetField;
private final List<String> formats;
private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
private final String outputFormat;

DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField) {
this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT);
}

DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField, String outputFormat) {
super(tag, description);
this.timezone = timezone;
this.locale = locale;
Expand All @@ -65,6 +72,8 @@ public final class DateProcessor extends AbstractProcessor {
DateFormat dateFormat = DateFormat.fromString(format);
dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
}
this.outputFormat = outputFormat;
formatter = DateFormatter.forPattern(this.outputFormat);
}

private ZoneId newDateTimeZone(Map<String, Object> params) {
Expand Down Expand Up @@ -99,7 +108,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
}

ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime));
ingestDocument.setFieldValue(targetField, formatter.format(dateTime));
return ingestDocument;
}

Expand Down Expand Up @@ -128,6 +137,10 @@ List<String> getFormats() {
return formats;
}

String getOutputFormat() {
return outputFormat;
}

public static final class Factory implements Processor.Factory {

private final ScriptService scriptService;
Expand All @@ -153,8 +166,16 @@ public DateProcessor create(Map<String, Processor.Factory> registry, String proc
"locale", localeString, scriptService);
}
List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
String outputFormat =
ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
try {
DateFormatter.forPattern(outputFormat);
} catch (Exception e) {
throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e);
}

return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
targetField);
targetField, outputFormat);
}
}
}
38 changes: 38 additions & 0 deletions ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,42 @@ public void testParseTargetField() throws Exception {
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getTargetField(), equalTo(targetField));
}

public void testParseOutputFormat() throws Exception {
final String outputFormat = "dd:MM:yyyy";
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getOutputFormat(), equalTo(outputFormat));
}

public void testDefaultOutputFormat() throws Exception {
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
DateProcessor processor = factory.create(null, null, null, config);
assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
}

public void testInvalidOutputFormatRejected() throws Exception {
final String outputFormat = "invalid_date_format";
Map<String, Object> config = new HashMap<>();
String sourceField = randomAlphaOfLengthBetween(1, 10);
String targetField = randomAlphaOfLengthBetween(1, 10);
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);

IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
}
}
14 changes: 14 additions & 0 deletions modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.test.ESTestCase;

import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
Expand Down Expand Up @@ -224,4 +225,17 @@ null, templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Fac
assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid"));
}

public void testOutputFormat() {
long nanosAfterEpoch = randomLongBetween(1, 999999);
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), null, null, null,
"date_as_string", Collections.singletonList("iso8601"), "date_as_date", "HH:mm:ss.SSSSSSSSS");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", Instant.EPOCH.plusNanos(nanosAfterEpoch).toString());
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
// output format is time only with nanosecond precision
String expectedDate = "00:00:00." + String.format(Locale.ROOT, "%09d", nanosAfterEpoch);
assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate));
}
}

0 comments on commit eeeb355

Please sign in to comment.