QA: Test template creation during rolling restart

Adds a test that we create the appropriate x-pack templates during the
rolling restart from the pre-6.2 OSS-zip distribution to the new zip
distribution that contains xpack. This is one way to answer the question
"does xpack acting sanely during the rolling upgrade and after it?" It
isn't as good as full exercising xpack but it is fairly simple and would
have caught elastic#30832.

Relates to elastic#30731

qa/rolling-upgrade/build.gradle

@@ -72,6 +72,7 @@ for (Version version : bwcVersions.wireCompatible) {
   Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
   oldClusterTestRunner.configure {
     systemProperty 'tests.rest.suite', 'old_cluster'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
   }
 
   Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
@@ -96,6 +97,7 @@ for (Version version : bwcVersions.wireCompatible) {
   oneThirdUpgradedTestRunner.configure {
     systemProperty 'tests.rest.suite', 'mixed_cluster'
     systemProperty 'tests.first_round', 'true'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
   }
 
@@ -108,6 +110,7 @@ for (Version version : bwcVersions.wireCompatible) {
   twoThirdsUpgradedTestRunner.configure {
     systemProperty 'tests.rest.suite', 'mixed_cluster'
     systemProperty 'tests.first_round', 'false'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
   }
 
@@ -119,6 +122,7 @@ for (Version version : bwcVersions.wireCompatible) {
   Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
   upgradedClusterTestRunner.configure {
     systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     /*
      * Force stopping all the upgraded nodes after the test runner
      * so they are alive during the test.

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java

@@ -19,15 +19,22 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.junit.Before;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.junit.Assume.assumeThat;
 
 /**
@@ -36,11 +43,9 @@
  */
 public class XPackIT extends AbstractRollingTestCase {
     @Before
-    public void skipIfNotXPack() {
+    public void skipIfNotZip() {
         assumeThat("test is only supported if the distribution contains xpack",
                 System.getProperty("tests.distribution"), equalTo("zip"));
-        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
-                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
         /*
          * *Mostly* we want this for when we're upgrading from pre-6.3's
          * zip distribution which doesn't contain xpack to post 6.3's zip
@@ -50,11 +55,81 @@ public void skipIfNotXPack() {
     }
 
     /**
-     * Test a basic feature (SQL) which doesn't require any trial license.
-     * Note that the test methods on this class can run in any order so we
-     * <strong>might</strong> have already installed a trial license.
+     * Tests that xpack is able to work itself into a sane state during the
+     * upgrade by testing that it is able to create all of the templates that
+     * it needs. This isn't a very strong assertion of sanity, but it is better
+     * than nothing and should catch a few sad cases.
+     * <p>
+     * The trouble is that when xpack isn't able to create the templates that
+     * it needs it retries over and over and over again. This can
+     * <strong>really</strong> slow things down. This test asserts that xpack
+     * was able to create the templates so it <strong>shouldn't</strong> be
+     * spinning trying to create things and slowing down the rest of the
+     * system.
+     */
+    public void testIndexTemplatesCreated() throws Exception {
+        Version upgradeFromVersion =
+                Version.fromString(System.getProperty("tests.upgrade_from_version"));
+        boolean upgradeFromVersionHasXPack = upgradeFromVersion.onOrAfter(Version.V_6_3_0);
+        assumeFalse("this test doesn't really prove anything if the starting version has xpack and it is *much* more complex to maintain",
+                upgradeFromVersionHasXPack);
+        assumeFalse("since we're upgrading from a version without x-pack it won't have any templates",
+                CLUSTER_TYPE == ClusterType.OLD);
+
+        List<String> expectedTemplates = new ArrayList<>();
+        // Watcher creates its templates as soon as the first watcher node connects
+        expectedTemplates.add(".triggered_watches");
+        expectedTemplates.add(".watch-history-8");
+        expectedTemplates.add(".watches");
+        if (masterIsNewVersion()) {
+            // Everything else waits until the master is upgraded to create its templates
+            expectedTemplates.add(".ml-anomalies-");
+            expectedTemplates.add(".ml-meta");
+            expectedTemplates.add(".ml-notifications");
+            expectedTemplates.add(".ml-state");
+            expectedTemplates.add("logstash-index-template");
+            expectedTemplates.add("security-index-template");
+            expectedTemplates.add("security_audit_log");
+        }
+        Collections.sort(expectedTemplates);
+
+        /*
+         * The index templates are created asynchronously after startup and
+         * while this is usually fast we use assertBusy here just in case
+         * they aren't created by the time this test is run.
+         */
+        assertBusy(() -> {
+            List<String> actualTemplates;
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    client().performRequest(new Request("GET", "/_template")).getEntity().getContent())) {
+                actualTemplates = new ArrayList<>(parser.map().keySet());
+            }
+            Collections.sort(actualTemplates);
+            /*
+             * This test asserts that the templates match *exactly* to force
+             * us to keep the list of templates up to date. Most templates
+             * aren't likely to cause a problem on upgrade but it is better
+             * to be safe and make sure they are all created than to be sorry
+             * and miss a bug that causes one to be missed on upgrade.
+             *
+             * We sort the templates so the error message is easy to read.
+             */
+            assertEquals(expectedTemplates, actualTemplates);
+        });
+    }
+
+    /**
+     * Test a basic feature (SQL) after the upgrade which only requires the
+     * "default" basic license. Note that the test methods on this class can
+     * run in any order so we <strong>might</strong> have already installed a
+     * trial license.
      */
-    public void testBasicFeature() throws IOException {
+    public void testBasicFeatureAfterUpgrade() throws IOException {
+        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
+                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
+
         Request bulk = new Request("POST", "/sql_test/doc/_bulk");
         bulk.setJsonEntity(
               "{\"index\":{}}\n"
@@ -71,16 +146,20 @@ public void testBasicFeature() throws IOException {
     }
 
     /**
-     * Test creating a trial license and using it. This is interesting because
-     * our other tests test cover starting a new cluster with the default
-     * distribution and enabling the trial license but this test is the only
-     * one that can upgrade from the oss distribution to the default
-     * distribution with xpack and the create a trial license. We don't
-     * <strong>do</strong> a lot with the trial license because for the most
-     * part those things are tested elsewhere, off in xpack. But we do use the
-     * trial license a little bit to make sure that it works.
+     * Test creating a trial license after the upgrade and a feature (ML) that
+     * requires the license. Our other tests test cover starting a new cluster
+     * with the default distribution and enabling the trial license but this
+     * test is the only one tests the rolling upgrade from the oss distribution
+     * to the default distribution with xpack and then creating of a trial
+     * license. We don't <strong>do</strong> a lot with the trial license
+     * because for the most part those things are tested elsewhere, off in
+     * xpack. But we do use the trial license a little bit to make sure that
+     * creating it worked properly.
      */
     public void testTrialLicense() throws IOException {
+        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
+                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
+
         Request startTrial = new Request("POST", "/_xpack/license/start_trial");
         startTrial.addParameter("acknowledge", "true");
         client().performRequest(startTrial);
@@ -108,4 +187,22 @@ public void testTrialLicense() throws IOException {
                 + "}\n");
         client().performRequest(createJob);
     }
+
+    /**
+     * Has the master been upgraded to the new version?
+     */
+    private boolean masterIsNewVersion() throws IOException {
+        Map<?, ?> map;
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                client().performRequest(new Request("GET", "/_nodes/_master")).getEntity().getContent())) {
+            map = parser.map();
+        }
+        map = (Map<?, ?>) map.get("nodes");
+        assertThat(map.values(), hasSize(1));
+        map = (Map<?, ?>) map.values().iterator().next();
+        Version masterVersion = Version.fromString(map.get("version").toString());
+        return Version.CURRENT.equals(masterVersion);
+    }
 }