Switch mapping/aggregations over to java time (#36363)

This commit moves the aggregation and mapping code from joda time to
java time. This includes field mappers, root object mappers, aggregations with date
histograms, query builders and a lot of changes within tests.

The cut-over to java time is a requirement so that we can support nanoseconds
properly in a future field mapper.

Relates #27330

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -55,4 +55,3 @@ public TemporalAccessor parseJodaDate() {
         return jodaFormatter.parse("1234567890");
     }
 }
-

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java

@@ -31,9 +31,9 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.List;
 import java.util.Map;
 
@@ -107,7 +107,7 @@ public String typeName() {
         }
 
         @Override
-        public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
+        public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
             throw new UnsupportedOperationException(
                 "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations");
         }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -59,10 +59,10 @@
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -301,7 +301,7 @@ public Object valueForDisplay(Object value) {
         }
 
         @Override
-        public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
+        public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
             if (timeZone != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName()
                     + "] does not support custom time zones");

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java

@@ -31,9 +31,9 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.List;
 import java.util.Map;
 
@@ -107,7 +107,7 @@ public String typeName() {
         }
 
         @Override
-        public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
+        public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
             throw new UnsupportedOperationException(
                 "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations");
         }

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -46,9 +46,9 @@
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -208,7 +208,7 @@ public BytesRef parseBytesRef(String value) {
         };
 
         @Override
-        public DocValueFormat docValueFormat(final String format, final DateTimeZone timeZone) {
+        public DocValueFormat docValueFormat(final String format, final ZoneId timeZone) {
             return COLLATE_FORMAT;
         }
     }

qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml

@@ -9,7 +9,7 @@
         index: timetest
         body:
           mappings:
-            test: { "properties": { "my_time": {"type": "date"}}}
+            test: { "properties": { "my_time": {"type": "date", "format": "strict_date_optional_time_nanos"}}}
 
   - do:
       ingest.put_pipeline:

server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -368,7 +367,7 @@ public static final class Tombstone implements ToXContentObject, Writeable {
             TOMBSTONE_PARSER.declareString((b, s) -> {}, new ParseField(DELETE_DATE_KEY));
         }
 
-        static final DateFormatter FORMATTER = DateFormatters.forPattern("strict_date_optional_time").withZone(ZoneOffset.UTC);
+        static final DateFormatter FORMATTER = DateFormatter.forPattern("strict_date_optional_time").withZone(ZoneOffset.UTC);
 
         static ContextParser<Void, Tombstone> getParser() {
             return (parser, context) -> TOMBSTONE_PARSER.apply(parser, null).build();

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -28,8 +28,8 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.time.DateMathParser;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -819,7 +819,7 @@ private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options
 
     static final class DateMathExpressionResolver implements ExpressionResolver {
 
-        private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatters.forPattern("uuuu.MM.dd");
+        private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
         private static final String EXPRESSION_LEFT_BOUND = "<";
         private static final String EXPRESSION_RIGHT_BOUND = ">";
         private static final char LEFT_BOUND = '{';
@@ -912,18 +912,19 @@ String resolveExpression(String expression, final Context context) {
                                     int formatPatternTimeZoneSeparatorIndex = patternAndTZid.indexOf(TIME_ZONE_BOUND);
                                     if (formatPatternTimeZoneSeparatorIndex != -1) {
                                         dateFormatterPattern = patternAndTZid.substring(0, formatPatternTimeZoneSeparatorIndex);
-                                        timeZone = ZoneId.of(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1));
+                                        timeZone = DateUtils.of(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1));
                                     } else {
                                         dateFormatterPattern = patternAndTZid;
                                         timeZone = ZoneOffset.UTC;
                                     }
-                                    dateFormatter = DateFormatters.forPattern(dateFormatterPattern);
+                                    dateFormatter = DateFormatter.forPattern(dateFormatterPattern);
                                 }
+
                                 DateFormatter formatter = dateFormatter.withZone(timeZone);
                                 DateMathParser dateMathParser = formatter.toDateMathParser();
-                                long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);
+                                Instant instant = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);
 
-                                String time = formatter.format(Instant.ofEpochMilli(millis));
+                                String time = formatter.format(instant);
                                 beforePlaceHolderSb.append(time);
                                 inPlaceHolderSb = new StringBuilder();
                                 inPlaceHolder = false;

server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -48,7 +47,7 @@
  */
 public final class UnassignedInfo implements ToXContentFragment, Writeable {
 
-    public static final DateFormatter DATE_TIME_FORMATTER = DateFormatters.forPattern("dateOptionalTime").withZone(ZoneOffset.UTC);
+    public static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("dateOptionalTime").withZone(ZoneOffset.UTC);
 
     public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
         Setting.positiveTimeSetting("index.unassigned.node_left.delayed_timeout", TimeValue.timeValueMinutes(1), Property.Dynamic,

server/src/main/java/org/elasticsearch/common/Rounding.java

@@ -19,9 +19,11 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
@@ -188,7 +190,7 @@ static class TimeUnitRounding extends Rounding {
 
         TimeUnitRounding(StreamInput in) throws IOException {
             unit = DateTimeUnit.resolve(in.readByte());
-            timeZone = ZoneId.of(in.readString());
+            timeZone = DateUtils.of(in.readString());
             unitRoundsToMidnight = unit.getField().getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L;
         }
 
@@ -367,8 +369,11 @@ public long nextRoundingValue(long utcMillis) {
         @Override
         public void innerWriteTo(StreamOutput out) throws IOException {
             out.writeByte(unit.getId());
-            String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
-            out.writeString(tz);
+            if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
+                out.writeString(timeZone.getId());
+            } else {
+                out.writeString(DateUtils.zoneIdToDateTimeZone(timeZone).getID());
+            }
         }
 
         @Override
@@ -417,7 +422,7 @@ public String toString() {
 
         TimeIntervalRounding(StreamInput in) throws IOException {
             interval = in.readVLong();
-            timeZone = ZoneId.of(in.readString());
+            timeZone = DateUtils.of(in.readString());
         }
 
         @Override
@@ -490,8 +495,11 @@ public long nextRoundingValue(long time) {
         @Override
         public void innerWriteTo(StreamOutput out) throws IOException {
             out.writeVLong(interval);
-            String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
-            out.writeString(tz);
+            if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
+                out.writeString(timeZone.getId());
+            } else {
+                out.writeString(DateUtils.zoneIdToDateTimeZone(timeZone).getID());
+            }
         }
 
         @Override

server/src/main/java/org/elasticsearch/common/Table.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateFormatters;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -85,7 +84,7 @@ public Table endHeaders() {
         return this;
     }
 
-    private static final DateFormatter FORMATTER = DateFormatters.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
+    private static final DateFormatter FORMATTER = DateFormatter.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
 
     public Table startRow() {
         if (headers.isEmpty()) {

server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -653,6 +653,23 @@ public DateTimeZone readOptionalTimeZone() throws IOException {
         return null;
     }
 
+    /**
+     * Read a {@linkplain DateTimeZone}.
+     */
+    public ZoneId readZoneId() throws IOException {
+        return ZoneId.of(readString());
+    }
+
+    /**
+     * Read an optional {@linkplain ZoneId}.
+     */
+    public ZoneId readOptionalZoneId() throws IOException {
+        if (readBoolean()) {
+            return ZoneId.of(readString());
+        }
+        return null;
+    }
+
     public int[] readIntArray() throws IOException {
         int length = readArraySize();
         int[] values = new int[length];

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -55,6 +55,7 @@
 import java.nio.file.FileSystemLoopException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
+import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.Collection;
 import java.util.Collections;
@@ -677,7 +678,6 @@ public final <K, V> void writeMap(final Map<K, V> map, final Writer<K> keyWriter
         writers.put(ZonedDateTime.class, (o, v) -> {
             o.writeByte((byte) 23);
             final ZonedDateTime zonedDateTime = (ZonedDateTime) v;
-            zonedDateTime.getZone().getId();
             o.writeString(zonedDateTime.getZone().getId());
             o.writeLong(zonedDateTime.toInstant().toEpochMilli());
         });
@@ -988,6 +988,13 @@ public void writeTimeZone(DateTimeZone timeZone) throws IOException {
         writeString(timeZone.getID());
     }
 
+    /**
+     * Write a {@linkplain ZoneId} to the stream.
+     */
+    public void writeZoneId(ZoneId timeZone) throws IOException {
+        writeString(timeZone.getId());
+    }
+
     /**
      * Write an optional {@linkplain DateTimeZone} to the stream.
      */
@@ -1000,6 +1007,18 @@ public void writeOptionalTimeZone(@Nullable DateTimeZone timeZone) throws IOExce
         }
     }
 
+    /**
+     * Write an optional {@linkplain ZoneId} to the stream.
+     */
+    public void writeOptionalZoneId(@Nullable ZoneId timeZone) throws IOException {
+        if (timeZone == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeZoneId(timeZone);
+        }
+    }
+
     /**
      * Writes a list of {@link Streamable} objects
      */

server/src/main/java/org/elasticsearch/common/joda/JodaDateFormatter.java

@@ -31,12 +31,12 @@
 import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAccessor;
 import java.util.Locale;
+import java.util.Objects;
 
 public class JodaDateFormatter implements DateFormatter {
-    final String pattern;
 
+    final String pattern;
     final DateTimeFormatter parser;
-
     final DateTimeFormatter printer;
 
     public JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
@@ -108,4 +108,21 @@ public ZoneId zone() {
     public DateMathParser toDateMathParser() {
         return new JodaDateMathParser(this);
     }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(locale(), zone(), pattern());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj.getClass().equals(this.getClass()) == false) {
+            return false;
+        }
+        JodaDateFormatter other = (JodaDateFormatter) obj;
+
+        return Objects.equals(pattern(), other.pattern()) &&
+            Objects.equals(locale(), other.locale()) &&
+            Objects.equals(zone(), other.zone());
+    }
 }

server/src/main/java/org/elasticsearch/common/joda/JodaDateMathParser.java

@@ -26,6 +26,7 @@
 import org.joda.time.MutableDateTime;
 import org.joda.time.format.DateTimeFormatter;
 
+import java.time.Instant;
 import java.time.ZoneId;
 import java.util.Objects;
 import java.util.function.LongSupplier;
@@ -50,7 +51,7 @@ public JodaDateMathParser(JodaDateFormatter dateTimeFormatter) {
     // if it has been used. For instance, the request cache does not cache requests that make
     // use of `now`.
     @Override
-    public long parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
+    public Instant parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
         final DateTimeZone timeZone = tz == null ? null : DateUtils.zoneIdToDateTimeZone(tz);
         long time;
         String mathString;
@@ -64,13 +65,13 @@ public long parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
         } else {
             int index = text.indexOf("||");
             if (index == -1) {
-                return parseDateTime(text, timeZone, roundUp);
+                return Instant.ofEpochMilli(parseDateTime(text, timeZone, roundUp));
             }
             time = parseDateTime(text.substring(0, index), timeZone, false);
             mathString = text.substring(index + 2);
         }
 
-        return parseMath(mathString, time, roundUp, timeZone);
+        return Instant.ofEpochMilli(parseMath(mathString, time, roundUp, timeZone));
     }
 
     private long parseMath(String mathString, long time, boolean roundUp, DateTimeZone timeZone) throws ElasticsearchParseException {