Merge remote-tracking branch 'elastic/master' into default-copy-setti…

…ngs-to-true

* elastic/master:
  Tests: Fail if test watches could not be triggered (elastic#30392)
  [ML] add version information in case of crash of native ML process (elastic#30674)
  Make TransportClusterStateAction abide to our style (elastic#30697)
  Change required version for Get Settings transport API changes to 6.4.0 (elastic#30706)
  [DOCS] Fixes edit URLs for stack overview (elastic#30583)
  Silence sleep based watcher test
  [TEST] Adjust version skips for movavg/movfn tests

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml

@@ -2,7 +2,7 @@
 #
 setup:
   - skip:
-        version: " - 6.4.0"
+        version: " - 6.3.99"
         reason:  "moving_fn added in 6.4.0"
   - do:
       indices.create:

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/250_moving_fn.yml

@@ -1,6 +1,6 @@
 setup:
   - skip:
-        version: " - 6.4.0"
+        version: " - 6.3.99"
         reason:  "moving_fn added in 6.4.0"
 
 ---

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml

@@ -197,7 +197,7 @@ setup:
 "Test typed keys parameter for date_histogram aggregation and max_bucket pipeline aggregation":
   - skip:
       features: warnings
-      version: " - 6.4.0"
+      version: " - 6.3.99"
       reason:  "deprecation added in 6.4.0"
   - do:
       warnings:

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -115,9 +115,9 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS
                 mdBuilder = MetaData.builder(currentState.metaData());
             }
 
-            // Filter our metadata that shouldn't be returned by API
-            for(ObjectObjectCursor<String, Custom> custom : currentState.metaData().customs()) {
-                if(!custom.value.context().contains(MetaData.XContentContext.API)) {
+            // filter out metadata that shouldn't be returned by the API
+            for (ObjectObjectCursor<String, Custom> custom : currentState.metaData().customs()) {
+                if (custom.value.context().contains(MetaData.XContentContext.API) == false) {
                     mdBuilder.removeCustom(custom.key);
                 }
             }

server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java

@@ -71,7 +71,7 @@ public GetSettingsRequest(StreamInput in) throws IOException {
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         names = in.readStringArray();
         humanReadable = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
             includeDefaults = in.readBoolean();
         }
     }
@@ -83,7 +83,7 @@ public void writeTo(StreamOutput out) throws IOException {
         indicesOptions.writeIndicesOptions(out);
         out.writeStringArray(names);
         out.writeBoolean(humanReadable);
-        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
             out.writeBoolean(includeDefaults);
         }
     }

server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java

@@ -114,7 +114,7 @@ public void readFrom(StreamInput in) throws IOException {
         }
         ImmutableOpenMap.Builder<String, Settings> defaultSettingsBuilder = ImmutableOpenMap.builder();
 
-        if (in.getVersion().onOrAfter(org.elasticsearch.Version.V_7_0_0_alpha1)) {
+        if (in.getVersion().onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
             int defaultSettingsSize = in.readVInt();
             for (int i = 0; i < defaultSettingsSize ; i++) {
                 defaultSettingsBuilder.put(in.readString(), Settings.readSettingsFromStream(in));
@@ -132,7 +132,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(cursor.key);
             Settings.writeSettingsToStream(cursor.value, out);
         }
-        if (out.getVersion().onOrAfter(org.elasticsearch.Version.V_7_0_0_alpha1)) {
+        if (out.getVersion().onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
             out.writeVInt(indexToDefaultSettings.size());
             for (ObjectObjectCursor<String, Settings> cursor : indexToDefaultSettings) {
                 out.writeString(cursor.key);

x-pack/docs/en/ml/index.asciidoc

@@ -17,11 +17,20 @@ from {es} for analysis and anomaly results are displayed in {kib} dashboards.
 
 --
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/overview.asciidoc
 include::overview.asciidoc[]
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/getting-started.asciidoc
 include::getting-started.asciidoc[]
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/configuring.asciidoc
 include::configuring.asciidoc[]
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/stopping-ml.asciidoc
 include::stopping-ml.asciidoc[]
-// include::ml-scenarios.asciidoc[]
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/api-quickref.asciidoc
 include::api-quickref.asciidoc[]
-//include::troubleshooting.asciidoc[]  Referenced from x-pack/docs/public/xpack-troubleshooting.asciidoc
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/functions.asciidoc
 include::functions.asciidoc[]

x-pack/docs/en/security/index.asciidoc

@@ -95,20 +95,29 @@ Head over to our {security-forum}[Security Discussion Forum]
 to share your experience, questions, and suggestions.
 --
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/getting-started.asciidoc
 include::getting-started.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/how-security-works.asciidoc
 include::how-security-works.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/authentication/overview.asciidoc
 include::authentication/overview.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/authorization/overview.asciidoc
 include::authorization/overview.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/auditing.asciidoc
 include::auditing.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/securing-communications.asciidoc
 include::securing-communications.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/using-ip-filtering.asciidoc
 include::using-ip-filtering.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/ccs-clients-integrations.asciidoc
 include::ccs-clients-integrations.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/security/reference.asciidoc
 include::reference.asciidoc[]

x-pack/docs/en/watcher/index.asciidoc

@@ -65,24 +65,35 @@ from the query, whether the condition was met, and what actions were taken.
 
 --
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/getting-started.asciidoc
 include::getting-started.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/how-watcher-works.asciidoc
 include::how-watcher-works.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/encrypting-data.asciidoc
 include::encrypting-data.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/input.asciidoc
 include::input.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/trigger.asciidoc
 include::trigger.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/condition.asciidoc
 include::condition.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/actions.asciidoc
 include::actions.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/transform.asciidoc
 include::transform.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/java.asciidoc
 include::java.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/managing-watches.asciidoc
 include::managing-watches.asciidoc[]
 
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/watcher/example-watches.asciidoc
 include::example-watches.asciidoc[]

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java

@@ -22,8 +22,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeoutException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 
 /**
@@ -84,20 +82,7 @@ public long getPid() throws TimeoutException {
     }
 
     public Map<String, Object> getNativeCodeInfo() throws TimeoutException {
-        String copyrightMessage = cppLogHandler.getCppCopyright(CONTROLLER_CONNECT_TIMEOUT);
-        Matcher matcher = Pattern.compile("Version (.+) \\(Build ([^)]+)\\) Copyright ").matcher(copyrightMessage);
-        if (matcher.find()) {
-            Map<String, Object> info = new HashMap<>(2);
-            info.put("version", matcher.group(1));
-            info.put("build_hash", matcher.group(2));
-            return info;
-        } else {
-            // If this happens it probably means someone has changed the format in lib/ver/CBuildInfo.cc
-            // in the machine-learning-cpp repo without changing the pattern above to match
-            String msg = "Unexpected native controller process copyright format: " + copyrightMessage;
-            LOGGER.error(msg);
-            throw new ElasticsearchException(msg);
-        }
+        return cppLogHandler.getNativeCodeInfo(CONTROLLER_CONNECT_TIMEOUT);
     }
 
     public void startProcess(List<String> command) throws IOException {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessageHandler.java

@@ -8,7 +8,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -30,10 +30,15 @@
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Deque;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Handle a stream of C++ log messages that arrive via a named pipe in JSON format.
@@ -181,6 +186,26 @@ public String getCppCopyright(Duration timeout) throws TimeoutException {
         return cppCopyright;
     }
 
+    /**
+     * Extracts version information from the copyright string which assumes a certain format.
+     */
+    public Map<String, Object> getNativeCodeInfo(Duration timeout) throws TimeoutException {
+        String copyrightMessage = getCppCopyright(timeout);
+        Matcher matcher = Pattern.compile("Version (.+) \\(Build ([^)]+)\\) Copyright ").matcher(copyrightMessage);
+        if (matcher.find()) {
+            Map<String, Object> info = new HashMap<>(2);
+            info.put("version", matcher.group(1));
+            info.put("build_hash", matcher.group(2));
+            return info;
+        } else {
+            // If this happens it probably means someone has changed the format in lib/ver/CBuildInfo.cc
+            // in the ml-cpp repo without changing the pattern above to match
+            String msg = "Unexpected native process copyright format: " + copyrightMessage;
+            LOGGER.error(msg);
+            throw new ElasticsearchException(msg);
+        }
+    }
+
     /**
      * Expected to be called very infrequently.
      */
@@ -281,8 +306,18 @@ private void parseMessage(XContent xContent, BytesReference bytesRef) {
         } catch (XContentParseException e) {
             String upstreamMessage = "Fatal error: '" + bytesRef.utf8ToString() + "'";
             if (upstreamMessage.contains("bad_alloc")) {
-                upstreamMessage += ", process ran out of memory.";
+                upstreamMessage += ", process ran out of memory";
             }
+
+            // add version information, so it's conveniently next to the crash log
+            upstreamMessage += ", version: ";
+            try {
+                Map<String, Object> versionInfo = getNativeCodeInfo(Duration.ofMillis(10));
+                upstreamMessage += String.format(Locale.ROOT, "%s (build %s)", versionInfo.get("version"), versionInfo.get("build_hash"));
+            } catch (TimeoutException timeoutException) {
+                upstreamMessage += "failed to retrieve";
+            }
+
             storeError(upstreamMessage);
             seenFatalError = true;
         } catch (IOException e) {

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.test;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
@@ -70,10 +71,12 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -177,7 +180,7 @@ protected boolean timeWarped() {
     public void _setup() throws Exception {
         if (timeWarped()) {
             timeWarp = new TimeWarp(internalCluster().getInstances(ScheduleTriggerEngineMock.class),
-                    (ClockMock)getInstanceFromMaster(Clock.class));
+                    (ClockMock)getInstanceFromMaster(Clock.class), logger);
         }
 
         if (internalCluster().size() > 0) {
@@ -536,24 +539,28 @@ public EmailSent send(Email email, Authentication auth, Profile profile, String
 
     protected static class TimeWarp {
 
-        protected final Iterable<ScheduleTriggerEngineMock> schedulers;
-        protected final ClockMock clock;
+        private final List<ScheduleTriggerEngineMock> schedulers;
+        private final ClockMock clock;
+        private final Logger logger;
 
-        public TimeWarp(Iterable<ScheduleTriggerEngineMock> schedulers, ClockMock clock) {
-            this.schedulers = schedulers;
+        TimeWarp(Iterable<ScheduleTriggerEngineMock> schedulers, ClockMock clock, Logger logger) {
+            this.schedulers = StreamSupport.stream(schedulers.spliterator(), false).collect(Collectors.toList());
             this.clock = clock;
+            this.logger = logger;
         }
 
         public void trigger(String jobName) {
-            schedulers.forEach(scheduler -> scheduler.trigger(jobName));
+            trigger(jobName, 1, null);
         }
 
         public ClockMock clock() {
             return clock;
         }
 
-        public void trigger(String id, int times, TimeValue timeValue) {
-            schedulers.forEach(scheduler -> scheduler.trigger(id, times, timeValue));
+        public void trigger(String watchId, int times, TimeValue timeValue) {
+            boolean isTriggered = schedulers.stream().anyMatch(scheduler -> scheduler.trigger(watchId, times, timeValue));
+            String msg = String.format(Locale.ROOT, "could not find watch [%s] to trigger", watchId);
+            assertThat(msg, isTriggered, is(true));
         }
     }
 

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java

@@ -47,8 +47,8 @@ public class ActivateWatchTests extends AbstractWatcherIntegrationTestCase {
     protected boolean timeWarped() {
         return false;
     }
-
-    // FIXME not to be sleep based
+    
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30699")
     public void testDeactivateAndActivate() throws Exception {
         PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
                 .setId("_id")

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java

@@ -77,18 +77,13 @@ public boolean remove(String jobId) {
         return watches.remove(jobId) != null;
     }
 
-    public void trigger(String jobName) {
-        trigger(jobName, 1, null);
+    public boolean trigger(String jobName) {
+        return trigger(jobName, 1, null);
     }
 
-    public void trigger(String jobName, int times) {
-        trigger(jobName, times, null);
-    }
-
-    public void trigger(String jobName, int times, TimeValue interval) {
+    public boolean trigger(String jobName, int times, TimeValue interval) {
         if (watches.containsKey(jobName) == false) {
-            logger.trace("not executing job [{}], not found", jobName);
-            return;
+            return false;
         }
 
         for (int i = 0; i < times; i++) {
@@ -108,5 +103,7 @@ public void trigger(String jobName, int times, TimeValue interval) {
                 }
             }
         }
+
+        return true;
     }
 }