[CCR] Add integration tests for follower indices

x-pack/plugins/cross_cluster_replication/common/constants/index.js

@@ -7,3 +7,4 @@
 export * from './plugin';
 export * from './base_path';
 export * from './app';
+export * from './settings';

x-pack/plugins/cross_cluster_replication/common/constants/settings.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const FOLLOWER_INDEX_ADVANCED_SETTINGS = {
+  maxReadRequestOperationCount: 5120,
+  maxOutstandingReadRequests: 12,
+  maxReadRequestSize: '32mb',
+  maxWriteRequestOperationCount: 5120,
+  maxWriteRequestSize: '9223372036854775807b',
+  maxOutstandingWriteRequests: 9,
+  maxWriteBufferCount: 2147483647,
+  maxWriteBufferSize: '512mb',
+  maxRetryDelay: '500ms',
+  readPollTimeout: '1m',
+};

x-pack/plugins/cross_cluster_replication/public/app/services/follower_index_default_settings.js

@@ -4,31 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-const defaultSettings = {
-  maxReadRequestOperationCount: 5120,
-  maxOutstandingReadRequests: 12,
-  maxReadRequestSize: '32mb',
-  maxWriteRequestOperationCount: 5120,
-  maxWriteRequestSize: '9223372036854775807b',
-  maxOutstandingWriteRequests: 9,
-  maxWriteBufferCount: 2147483647,
-  maxWriteBufferSize: '512mb',
-  maxRetryDelay: '500ms',
-  readPollTimeout: '1m',
-};
+import { FOLLOWER_INDEX_ADVANCED_SETTINGS } from '../../../common/constants';
 
 export const getSettingDefault = (name) => {
-  if(!defaultSettings[name]) {
+  if(!FOLLOWER_INDEX_ADVANCED_SETTINGS[name]) {
     throw new Error(`Unknown setting ${name}`);
   }
 
-  return defaultSettings[name];
+  return FOLLOWER_INDEX_ADVANCED_SETTINGS[name];
 };
 
 export const isSettingDefault = (name, value) => {
   return getSettingDefault(name) === value;
 };
 
 export const areAllSettingsDefault = (settings) => {
-  return Object.keys(defaultSettings).every((name) => isSettingDefault(name, settings[name]));
+  return Object.keys(FOLLOWER_INDEX_ADVANCED_SETTINGS).every((name) => isSettingDefault(name, settings[name]));
 };

x-pack/test/api_integration/apis/management/cross_cluster_replication/auto_follow_pattern.js

@@ -5,42 +5,21 @@
  */
 
 import expect from 'expect.js';
-import Chance from 'chance';
-import { API_BASE_PATH, REMOTE_CLUSTERS_API_BASE_PATH } from './constants';
 
-const chance = new Chance();
-const CHARS_POOL = 'abcdefghijklmnopqrstuvwxyz';
-const getRandomName = () => chance.string({ pool: CHARS_POOL });
-const CLUSTER_NAME = `test-${getRandomName()}`;
+import { API_BASE_PATH } from './constants';
+
+import { initClusterHelpers, getRandomString } from './lib';
+
 const AUTO_FOLLOW_PATTERNS_API_BASE_PATH = API_BASE_PATH + '/auto_follow_patterns';
 
 export default function ({ getService }) {
   let autoFollowPatternsCreated = [];
   const supertest = getService('supertest');
 
-  const addCluster = async (name = CLUSTER_NAME) => (
-    await supertest
-      .post(`${REMOTE_CLUSTERS_API_BASE_PATH}`)
-      .set('kbn-xsrf', 'xxx')
-      .send({
-        "name": name,
-        "seeds": [
-          "localhost:9300"
-        ],
-        "skipUnavailable": true,
-      })
-  );
-
-  const deleteCluster = (name = CLUSTER_NAME) => {
-    return (
-      supertest
-        .delete(`${REMOTE_CLUSTERS_API_BASE_PATH}/${name}`)
-        .set('kbn-xsrf', 'xxx')
-    );
-  };
+  const { CLUSTER_NAME, addCluster, deleteAllClusters } = initClusterHelpers(supertest);
 
-  const deleteAutoFollowPattern = async (name) => (
-    await supertest
+  const deleteAutoFollowPatternRequest = (name) => (
+    supertest
       .delete(`${AUTO_FOLLOW_PATTERNS_API_BASE_PATH}/${name}`)
       .set('kbn-xsrf', 'xxx')
   );
@@ -51,7 +30,7 @@ export default function ({ getService }) {
     followIndexPattern: '{{leader_index}}_follower'
   });
 
-  const createAutoFollowIndexRequest = (name = getRandomName(), payload = getAutoFollowIndexPayload()) => {
+  const createAutoFollowIndexRequest = (name = getRandomString(), payload = getAutoFollowIndexPayload()) => {
     autoFollowPatternsCreated.push(name);
 
     return supertest
@@ -61,14 +40,13 @@ export default function ({ getService }) {
   };
 
   const cleanUp = () => (
-    Promise.all([deleteCluster(), ...autoFollowPatternsCreated.map(name => deleteAutoFollowPattern(name))])
+    Promise.all([deleteAllClusters(), ...autoFollowPatternsCreated.map(deleteAutoFollowPatternRequest)])
       .then(() => {
         autoFollowPatternsCreated = [];
       })
   );
 
   describe('auto follow patterns', () => {
-
     afterEach(() => {
       return cleanUp();
     });
@@ -102,7 +80,7 @@ export default function ({ getService }) {
       it('should create an auto-follow pattern when cluster is known', async () => {
         await addCluster();
 
-        const name = getRandomName();
+        const name = getRandomString();
         const { body } = await createAutoFollowIndexRequest(name).expect(200);
 
         expect(body.acknowledged).to.eql(true);
@@ -111,7 +89,7 @@ export default function ({ getService }) {
 
     describe('get()', () => {
       it('should return a 404 when the auto-follow pattern is not found', async () => {
-        const name = getRandomName();
+        const name = getRandomString();
         const { body } = await supertest
           .get(`${AUTO_FOLLOW_PATTERNS_API_BASE_PATH}/${name}`)
           .expect(404);
@@ -120,7 +98,7 @@ export default function ({ getService }) {
       });
 
       it('should return an auto-follow pattern that was created', async () => {
-        const name = getRandomName();
+        const name = getRandomString();
         const autoFollowPattern = getAutoFollowIndexPayload();
 
         await addCluster();

x-pack/test/api_integration/apis/management/cross_cluster_replication/follower_indices.js

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+
+import { FOLLOWER_INDEX_ADVANCED_SETTINGS } from '../../../../../plugins/cross_cluster_replication/common/constants';
+import { API_BASE_PATH } from './constants';
+import { initClusterHelpers, initElasticsearchIndicesHelpers, getRandomString } from './lib';
+
+const FOLLOWER_INDICES_API_BASE_PATH = API_BASE_PATH + '/follower_indices';
+
+export default function ({ getService }) {
+  let followerIndicesCreated = [];
+  const supertest = getService('supertest');
+  const es = getService('es');
+
+  const { CLUSTER_NAME, addCluster, deleteAllClusters } = initClusterHelpers(supertest);
+  const { createIndex, deleteAllIndices } = initElasticsearchIndicesHelpers(es);
+
+  const getFollowerIndexPayload = (leaderIndexName = getRandomString(), remoteCluster = CLUSTER_NAME, advancedSettings = {}) => ({
+    remoteCluster,
+    leaderIndex: leaderIndexName,
+    ...advancedSettings,
+  });
+
+  const createFollowerIndexRequest = (name = getRandomString(), payload = getFollowerIndexPayload()) => {
+    followerIndicesCreated.push(name);
+
+    return supertest
+      .post(FOLLOWER_INDICES_API_BASE_PATH)
+      .set('kbn-xsrf', 'xxx')
+      .send({ ...payload, name });
+  };
+
+  const deleteFollowerIndexRequest = (name) => (
+    supertest
+      .put(`${FOLLOWER_INDICES_API_BASE_PATH}/${name}/unfollow`)
+      .set('kbn-xsrf', 'xxx')
+  );
+
+  const cleanUp = () => (
+    Promise.all([
+      deleteAllClusters(),
+      deleteAllIndices(),
+      ...followerIndicesCreated.map(deleteFollowerIndexRequest)
+    ]).then(() => { followerIndicesCreated = []; })
+  );
+
+  describe('follower indices', () => {
+    afterEach(() => cleanUp());
+
+    describe('list()', () => {
+      it('should return an empty array when there are no follower indices', async () => {
+        const { body } = await supertest
+          .get(FOLLOWER_INDICES_API_BASE_PATH)
+          .expect(200);
+
+        expect(body).to.eql({ indices: [] });
+      });
+    });
+
+    describe('create()', () => {
+      let payload;
+
+      beforeEach(async () => {
+        await addCluster();
+        payload = getFollowerIndexPayload();
+      });
+
+      it('should throw a 404 error when cluster is unknown', async () => {
+        payload.remoteCluster = 'unknown-cluster';
+
+        const { body } = await createFollowerIndexRequest(undefined, payload).expect(404);
+        expect(body.cause[0]).to.contain('no such remote cluster');
+      });
+
+      it('should throw a 404 error trying to follow an unknown index', async () => {
+        const { body } = await createFollowerIndexRequest(undefined, payload).expect(404);
+        expect(body.cause[0]).to.contain('no such index');
+      });
+
+      it('should create a follower index that follows an existing remote index', async () => {
+        // First let's create an index to follow
+        const leaderIndex = await createIndex();
+
+        const payload = getFollowerIndexPayload(leaderIndex);
+        const { body } = await createFollowerIndexRequest(undefined, payload).expect(200);
+
+        expect(body).to.eql({
+          follow_index_created: true,
+          follow_index_shards_acked: true,
+          index_following_started: true
+        });
+      });
+    });
+
+    describe('get()', () => {
+      beforeEach(async () => addCluster());
+
+      it('should return a 404 when the follower index does not exist', async () => {
+        const name = getRandomString();
+        const { body } = await supertest
+          .get(`${FOLLOWER_INDICES_API_BASE_PATH}/${name}`)
+          .expect(404);
+
+        expect(body.cause[0]).to.contain('no such index');
+      });
+
+      it('should return a follower index that was created', async () => {
+        const leaderIndex = await createIndex();
+
+        const name = getRandomString();
+        const payload = getFollowerIndexPayload(leaderIndex);
+        await createFollowerIndexRequest(name, payload).expect(200);
+
+        const { body } = await supertest
+          .get(`${FOLLOWER_INDICES_API_BASE_PATH}/${name}`)
+          .expect(200);
+
+        expect(body.leaderIndex).to.eql(leaderIndex);
+        expect(body.remoteCluster).to.eql(payload.remoteCluster);
+      });
+    });
+
+    describe('Advanced settings', () => {
+      beforeEach(() => addCluster());
+
+      it('hard-coded values should match Elasticsearch default values', async () => {
+        /**
+           * To make sure that the hard-coded values in the client match the default
+           * from Elasticsearch, we will create a follower index without any advanced settings.
+           * When we then retrieve the follower index it will have all the advanced settings
+           * coming from ES. We can then compare those settings with our hard-coded values.
+           */
+        const leaderIndex = await createIndex();
+
+        const name = getRandomString();
+        const payload = getFollowerIndexPayload(leaderIndex);
+        await createFollowerIndexRequest(name, payload).expect(200);
+
+        const { body } = await supertest.get(`${FOLLOWER_INDICES_API_BASE_PATH}/${name}`);
+
+        Object.entries(FOLLOWER_INDEX_ADVANCED_SETTINGS).forEach(([key, value]) => {
+          expect(value).to.eql(body[key]);
+        });
+      });
+    });
+  });
+}

x-pack/test/api_integration/apis/management/cross_cluster_replication/index.js

@@ -7,5 +7,6 @@
 export default function ({ loadTestFile }) {
   describe('cross cluster replication', () => {
     loadTestFile(require.resolve('./auto_follow_pattern'));
+    loadTestFile(require.resolve('./follower_indices'));
   });
 }

x-pack/test/api_integration/apis/management/cross_cluster_replication/lib/clusters.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { getRandomString } from './random';
+import { REMOTE_CLUSTERS_API_BASE_PATH } from '../constants';
+
+const CLUSTER_NAME = `test-${getRandomString()}`;
+
+/**
+ * Helpers for the CCR application to easily create and delete
+ * Remote clusters for the tests.
+ * @param {Supertest} supertest The supertest instance
+ */
+export const initClusterHelpers = (supertest) => {
+  let clusters = [];
+
+  const addCluster = (name = CLUSTER_NAME) => {
+    clusters.push(name);
+    return (
+      supertest
+        .post(`${REMOTE_CLUSTERS_API_BASE_PATH}`)
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          "name": name,
+          "seeds": [
+            "localhost:9300"
+          ],
+          "skipUnavailable": true,
+        })
+    );
+  };
+
+  const deleteCluster = (name = CLUSTER_NAME) => {
+    clusters = clusters.filter(c => c !== name);
+    return (
+      supertest
+        .delete(`${REMOTE_CLUSTERS_API_BASE_PATH}/${name}`)
+        .set('kbn-xsrf', 'xxx')
+    );
+  };
+
+  const deleteAllClusters = () => (
+    Promise.all(clusters.map(deleteCluster)).then(() => {
+      clusters = [];
+    })
+  );
+
+  return ({
+    CLUSTER_NAME,
+    addCluster,
+    deleteCluster,
+    deleteAllClusters,
+  });
+};