Date: Add DateFormatters class that uses java.time

build.gradle

@@ -445,12 +445,19 @@ allprojects {
   }
 
   File licenseHeaderFile;
-  if (eclipse.project.name.startsWith(':x-pack')) {
+  String prefix = ':x-pack';
+
+  if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+    prefix = prefix.replace(':', '_')
+  }
+  if (eclipse.project.name.startsWith(prefix)) {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/elastic-license-header.txt')
   } else {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
   }
-  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace('\n', '\\\\n')
+
+  String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
+  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
   task copyEclipseSettings(type: Copy) {
     // TODO: "package this up" for external builds
     from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings')

docs/reference/migration/migrate_7_0/api.asciidoc

@@ -75,3 +75,7 @@ will be for such settings to be copied on such operations. To enable users in
 `copy_settings` parameter was added on the REST layer. As this behavior will be
 the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in
 8.0.0.
+
+==== The deprecated stored script contexts have now been removed
+When putting stored scripts, support for storing them with the deprecated `template` context or without a context is
+now removed. Scripts must be stored using the `script` context as mentioned in the documentation.

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java

@@ -19,38 +19,41 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
 import java.util.Locale;
 import java.util.function.Function;
 
 enum DateFormat {
     Iso8601 {
         @Override
-        Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return ISODateTimeFormat.dateTimeParser().withZone(timezone)::parseDateTime;
+        Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
+            DateFormatter formatter = DateFormatters.forPattern("date_time_no_millis");
+            return (date) -> ZonedDateTime.from(formatter.parse(date)).withZoneSameInstant(timezone);
         }
     },
     Unix {
         @Override
-        Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return (date) -> new DateTime((long)(Double.parseDouble(date) * 1000), timezone);
+        Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
+            return (date) -> Instant.ofEpochMilli(((Double) (Double.parseDouble(date) * 1000)).longValue()).atZone(timezone);
         }
     },
     UnixMs {
         @Override
-        Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return (date) -> new DateTime(Long.parseLong(date), timezone);
+        Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
+            return (date) -> Instant.ofEpochMilli(Long.parseLong(date)).atZone(timezone);
         }
     },
     Tai64n {
         @Override
-        Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return (date) -> new DateTime(parseMillis(date), timezone);
+        Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
+            return (date) -> Instant.ofEpochMilli(parseMillis(date)).atZone(timezone);
         }
 
         private long parseMillis(String date) {
@@ -63,15 +66,20 @@ private long parseMillis(String date) {
             return ((base * 1000) - 10000) + (rest/1000000);
         }
     },
-    Joda {
+    Time {
         @Override
-        Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            DateTimeFormatter parser = DateTimeFormat.forPattern(format).withZone(timezone).withLocale(locale);
-            return text -> parser.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()).parseDateTime(text);
+        Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
+            DateFormatter formatter = DateFormatters.forPattern(format, locale).withZone(timezone);
+            return text -> {
+                TemporalAccessor accessor = formatter.parse(text);
+                ZonedDateTime startOfThisYear = DateFormatters.EPOCH_ZONED_DATE_TIME.withYear(ZonedDateTime.now(timezone)
+                    .get(ChronoField.YEAR));
+                return DateFormatters.toZonedDateTime(accessor, startOfThisYear).withZoneSameLocal(timezone);
+            };
         }
     };
 
-    abstract Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale);
+    abstract Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale);
 
     static DateFormat fromString(String format) {
         switch (format) {
@@ -84,7 +92,7 @@ static DateFormat fromString(String format) {
             case "TAI64N":
                 return Tai64n;
             default:
-                return Joda;
+                return Time;
         }
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java

@@ -19,6 +19,17 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.IllformedLocaleException;
@@ -27,16 +38,6 @@
 import java.util.Map;
 import java.util.function.Function;
 
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.ConfigurationUtils;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
 public final class DateIndexNameProcessor extends AbstractProcessor {
 
     public static final String TYPE = "date_index_name";
@@ -45,10 +46,10 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
     private final String indexNamePrefix;
     private final String dateRounding;
     private final String indexNameFormat;
-    private final DateTimeZone timezone;
-    private final List<Function<String, DateTime>> dateFormats;
+    private final ZoneId timezone;
+    private final List<Function<String, ZonedDateTime>> dateFormats;
 
-    DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
+    DateIndexNameProcessor(String tag, String field, List<Function<String, ZonedDateTime>> dateFormats, ZoneId timezone,
                            String indexNamePrefix, String dateRounding, String indexNameFormat) {
         super(tag);
         this.field = field;
@@ -69,9 +70,9 @@ public void execute(IngestDocument ingestDocument) throws Exception {
             date = obj.toString();
         }
 
-        DateTime dateTime = null;
+        ZonedDateTime dateTime = null;
         Exception lastException = null;
-        for (Function<String, DateTime> dateParser : dateFormats) {
+        for (Function<String, ZonedDateTime> dateParser : dateFormats) {
             try {
                 dateTime = dateParser.apply(date);
             } catch (Exception e) {
@@ -84,13 +85,14 @@ public void execute(IngestDocument ingestDocument) throws Exception {
             throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
         }
 
-        DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
+        DateFormatter formatter = DateFormatters.forPattern(indexNameFormat);
+        String zoneId = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.toString();
         StringBuilder builder = new StringBuilder()
                 .append('<')
                 .append(indexNamePrefix)
                     .append('{')
-                        .append(formatter.print(dateTime)).append("||/").append(dateRounding)
-                            .append('{').append(indexNameFormat).append('|').append(timezone).append('}')
+                        .append(formatter.format(dateTime)).append("||/").append(dateRounding)
+                            .append('{').append(indexNameFormat).append('|').append(zoneId).append('}')
                     .append('}')
                 .append('>');
         String dynamicIndexName  = builder.toString();
@@ -118,11 +120,11 @@ String getIndexNameFormat() {
         return indexNameFormat;
     }
 
-    DateTimeZone getTimezone() {
+    ZoneId getTimezone() {
         return timezone;
     }
 
-    List<Function<String, DateTime>> getDateFormats() {
+    List<Function<String, ZonedDateTime>> getDateFormats() {
         return dateFormats;
     }
 
@@ -133,7 +135,7 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
                                              Map<String, Object> config) throws Exception {
             String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
-            DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
+            ZoneId timezone = timezoneString == null ? ZoneOffset.UTC : ZoneId.of(timezoneString);
             Locale locale = Locale.ENGLISH;
             if (localeString != null) {
                 try {
@@ -144,9 +146,9 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
             }
             List<String> dateFormatStrings = ConfigurationUtils.readOptionalList(TYPE, tag, config, "date_formats");
             if (dateFormatStrings == null) {
-                dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+                dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSX");
             }
-            List<Function<String, DateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
+            List<Function<String, ZonedDateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
             for (String format : dateFormatStrings) {
                 DateFormat dateFormat = DateFormat.fromString(format);
                 dateFormats.add(dateFormat.getFunction(format, timezone, locale));

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java

@@ -28,18 +28,46 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.TemplateScript;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.ISODateTimeFormat;
 
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.SignStyle;
+import java.time.temporal.ChronoField;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 
+import static java.time.temporal.ChronoField.DAY_OF_MONTH;
+import static java.time.temporal.ChronoField.HOUR_OF_DAY;
+import static java.time.temporal.ChronoField.MILLI_OF_SECOND;
+import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
+import static java.time.temporal.ChronoField.MONTH_OF_YEAR;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+
 public final class DateProcessor extends AbstractProcessor {
 
+    private static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
+        .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+        .appendLiteral("-")
+        .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral('-')
+        .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral('T')
+        .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendLiteral(':')
+        .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
+        .appendFraction(MILLI_OF_SECOND, 3, 3, true)
+        .appendOffset("+HH:MM", "Z")
+        .toFormatter(Locale.ROOT);
+
+
     public static final String TYPE = "date";
     static final String DEFAULT_TARGET_FIELD = "@timestamp";
 
@@ -48,7 +76,7 @@ public final class DateProcessor extends AbstractProcessor {
     private final String field;
     private final String targetField;
     private final List<String> formats;
-    private final List<Function<Map<String, Object>, Function<String, DateTime>>> dateParsers;
+    private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
 
     DateProcessor(String tag, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
                   String field, List<String> formats, String targetField) {
@@ -65,8 +93,8 @@ public final class DateProcessor extends AbstractProcessor {
         }
     }
 
-    private DateTimeZone newDateTimeZone(Map<String, Object> params) {
-        return timezone == null ? DateTimeZone.UTC : DateTimeZone.forID(timezone.newInstance(params).execute());
+    private ZoneId newDateTimeZone(Map<String, Object> params) {
+        return timezone == null ? ZoneOffset.UTC : ZoneId.of(timezone.newInstance(params).execute());
     }
 
     private Locale newLocale(Map<String, Object> params) {
@@ -82,9 +110,9 @@ public void execute(IngestDocument ingestDocument) {
             value = obj.toString();
         }
 
-        DateTime dateTime = null;
+        ZonedDateTime dateTime = null;
         Exception lastException = null;
-        for (Function<Map<String, Object>, Function<String, DateTime>> dateParser : dateParsers) {
+        for (Function<Map<String, Object>, Function<String, ZonedDateTime>> dateParser : dateParsers) {
             try {
                 dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value);
             } catch (Exception e) {
@@ -97,7 +125,7 @@ public void execute(IngestDocument ingestDocument) {
             throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
         }
 
-        ingestDocument.setFieldValue(targetField, ISODateTimeFormat.dateTime().print(dateTime));
+        ingestDocument.setFieldValue(targetField, dateTime.format(DATE_FORMATTER));
     }
 
     @Override

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java

@@ -63,7 +63,7 @@ boolean isIgnoreMissing() {
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        List values = ingestDocument.getFieldValue(field, List.class, ignoreMissing);
+        List<?> values = ingestDocument.getFieldValue(field, List.class, ignoreMissing);
         if (values == null) {
             if (ignoreMissing) {
                 return;

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java

@@ -73,7 +73,9 @@ public RemoveProcessor create(Map<String, Processor.Factory> registry, String pr
             final List<String> fields = new ArrayList<>();
             final Object field = ConfigurationUtils.readObject(TYPE, processorTag, config, "field");
             if (field instanceof List) {
-                fields.addAll((List) field);
+                @SuppressWarnings("unchecked")
+                List<String> stringList = (List<String>) field;
+                fields.addAll(stringList);
             } else {
                 fields.add((String) field);
             }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java

@@ -49,6 +49,7 @@ public enum SortOrder {
             this.direction = direction;
         }
 
+        @Override
         public String toString() {
             return this.direction;
         }
@@ -94,13 +95,13 @@ String getTargetField() {
     @Override
     @SuppressWarnings("unchecked")
     public void execute(IngestDocument document) {
-        List<? extends Comparable> list = document.getFieldValue(field, List.class);
+        List<? extends Comparable<Object>> list = document.getFieldValue(field, List.class);
 
         if (list == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot sort.");
         }
 
-        List<? extends Comparable> copy = new ArrayList<>(list);
+        List<? extends Comparable<Object>> copy = new ArrayList<>(list);
 
         if (order.equals(SortOrder.ASCENDING)) {
             Collections.sort(copy);