[elasticsearch] patch mappings that are missing types (elastic#12783)

* [elasticsearch] patch mappings that are missing types * [elasticsearch/healthCheck] fix tests * fix doc typo * [tests/functional/dashboard] fix suite name * [es/healthCheck/ensureTypesExist] limit randomness a bit * [test/functional] update es archives with complete mappings
cjcenizal · Jul 12, 2017 · 929aa8e · 929aa8e
1 parent 8d1bbd9
commit 929aa8e
Show file tree

Hide file tree

Showing 11 changed files with 1,477 additions and 232 deletions.
diff --git a/src/core_plugins/elasticsearch/lib/__tests__/ensure_types_exist.js b/src/core_plugins/elasticsearch/lib/__tests__/ensure_types_exist.js
@@ -0,0 +1,247 @@
+import expect from 'expect.js';
+import sinon from 'sinon';
+import { cloneDeep } from 'lodash';
+import Chance from 'chance';
+
+import { ensureTypesExist } from '../ensure_types_exist';
+
+const chance = new Chance();
+
+function createRandomTypes(n = chance.integer({ min: 10, max: 20 })) {
+  return chance.n(
+    () => ({
+      name: chance.word(),
+      mapping: {
+        type: chance.pickone(['keyword', 'text', 'integer', 'boolean'])
+      }
+    }),
+    n
+  );
+}
+
+function typesToMapping(types) {
+  return types.reduce((acc, type) => ({
+    ...acc,
+    [type.name]: type.mapping
+  }), {});
+}
+
+function createV5Index(name, types) {
+  return {
+    [name]: {
+      mappings: typesToMapping(types)
+    }
+  };
+}
+
+function createV6Index(name, types) {
+  return {
+    [name]: {
+      mappings: {
+        doc: {
+          properties: typesToMapping(types)
+        }
+      }
+    }
+  };
+}
+
+function createCallCluster(index) {
+  return sinon.spy(async (method, params) => {
+    switch (method) {
+      case 'indices.get':
+        expect(params).to.have.property('index', Object.keys(index)[0]);
+        return cloneDeep(index);
+      case 'indices.putMapping':
+        return { ok: true };
+      default:
+        throw new Error(`stub not expecting callCluster('${method}')`);
+    }
+  });
+}
+
+describe('es/healthCheck/ensureTypesExist()', () => {
+  describe('general', () => {
+    it('reads the _mappings feature of the indexName', async () => {
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV5Index(indexName, []));
+      await ensureTypesExist({
+        callCluster,
+        indexName,
+        types: [],
+        log: sinon.stub()
+      });
+
+      sinon.assert.calledOnce(callCluster);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({
+        feature: '_mappings'
+      }));
+    });
+  });
+
+  describe('v5 index', () => {
+    it('does nothing if mappings match elasticsearch', async () => {
+      const types = createRandomTypes();
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV5Index(indexName, types));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types,
+        log: sinon.stub()
+      });
+
+      sinon.assert.calledOnce(callCluster);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+    });
+
+    it('adds types that are not in index', async () => {
+      const indexTypes = createRandomTypes();
+      const missingTypes = indexTypes.splice(-5);
+
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV5Index(indexName, indexTypes));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types: [
+          ...indexTypes,
+          ...missingTypes,
+        ],
+        log: sinon.stub()
+      });
+
+      sinon.assert.callCount(callCluster, 1 + missingTypes.length);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+      missingTypes.forEach(type => {
+        sinon.assert.calledWith(callCluster, 'indices.putMapping', sinon.match({
+          index: indexName,
+          type: type.name,
+          body: type.mapping
+        }));
+      });
+    });
+
+    it('ignores extra types in index', async () => {
+      const indexTypes = createRandomTypes();
+      const missingTypes = indexTypes.splice(-5);
+
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV5Index(indexName, indexTypes));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types: missingTypes,
+        log: sinon.stub()
+      });
+
+      sinon.assert.callCount(callCluster, 1 + missingTypes.length);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+      missingTypes.forEach(type => {
+        sinon.assert.calledWith(callCluster, 'indices.putMapping', sinon.match({
+          index: indexName,
+          type: type.name,
+          body: type.mapping
+        }));
+      });
+    });
+  });
+
+  describe('v6 index', () => {
+    it('does nothing if mappings match elasticsearch', async () => {
+      const types = createRandomTypes();
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV6Index(indexName, types));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types,
+        log: sinon.stub()
+      });
+
+      sinon.assert.calledOnce(callCluster);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+    });
+
+    it('adds types that are not in index', async () => {
+      const indexTypes = createRandomTypes();
+      const missingTypes = indexTypes.splice(-5);
+
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV6Index(indexName, indexTypes));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types: [
+          ...indexTypes,
+          ...missingTypes,
+        ],
+        log: sinon.stub()
+      });
+
+      sinon.assert.callCount(callCluster, 1 + missingTypes.length);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+      missingTypes.forEach(type => {
+        sinon.assert.calledWith(callCluster, 'indices.putMapping', sinon.match({
+          index: indexName,
+          type: 'doc',
+          body: {
+            properties: {
+              [type.name]: type.mapping,
+            }
+          }
+        }));
+      });
+    });
+
+    it('ignores extra types in index', async () => {
+      const indexTypes = createRandomTypes();
+      const missingTypes = indexTypes.splice(-5);
+
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV6Index(indexName, indexTypes));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types: missingTypes,
+        log: sinon.stub()
+      });
+
+      sinon.assert.callCount(callCluster, 1 + missingTypes.length);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+      missingTypes.forEach(type => {
+        sinon.assert.calledWith(callCluster, 'indices.putMapping', sinon.match({
+          index: indexName,
+          type: 'doc',
+          body: {
+            properties: {
+              [type.name]: type.mapping,
+            }
+          }
+        }));
+      });
+    });
+
+    it('does not define the _default_ type', async () => {
+      const indexTypes = [];
+      const missingTypes = [
+        {
+          name: '_default_',
+          mapping: {}
+        }
+      ];
+
+      const indexName = chance.word();
+      const callCluster = createCallCluster(createV6Index(indexName, indexTypes));
+      await ensureTypesExist({
+        indexName,
+        callCluster,
+        types: missingTypes,
+        log: sinon.stub()
+      });
+
+      sinon.assert.calledOnce(callCluster);
+      sinon.assert.calledWith(callCluster, 'indices.get', sinon.match({ index: indexName }));
+    });
+  });
+});
diff --git a/src/core_plugins/elasticsearch/lib/__tests__/health_check.js b/src/core_plugins/elasticsearch/lib/__tests__/health_check.js
@@ -11,6 +11,7 @@ import kibanaVersion from '../kibana_version';
 import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 import * as determineEnabledScriptingLangsNS from '../determine_enabled_scripting_langs';
 import { determineEnabledScriptingLangs } from '../determine_enabled_scripting_langs';
+import * as ensureTypesExistNS from '../ensure_types_exist';
 
 const esPort = esTestServerUrlParts.port;
 const esUrl = url.format(esTestServerUrlParts);
@@ -29,6 +30,7 @@ describe('plugins/elasticsearch', () => {
 
       // Stub the Kibana version instead of drawing from package.json.
       sinon.stub(kibanaVersion, 'get').returns(COMPATIBLE_VERSION_NUMBER);
+      sinon.stub(ensureTypesExistNS, 'ensureTypesExist');
 
       // setup the plugin stub
       plugin = {
@@ -85,6 +87,7 @@ describe('plugins/elasticsearch', () => {
     afterEach(() => {
       kibanaVersion.get.restore();
       determineEnabledScriptingLangs.restore();
+      ensureTypesExistNS.ensureTypesExist.restore();
     });
 
     it('should set the cluster green if everything is ready', function () {

diff --git a/src/core_plugins/elasticsearch/lib/ensure_types_exist.js b/src/core_plugins/elasticsearch/lib/ensure_types_exist.js
@@ -0,0 +1,68 @@
+/**
+ *  Checks that a kibana index has all of the types specified. Any type
+ *  that is not defined in the existing index will be added via the
+ *  `indicies.putMapping` API.
+ *
+ *  @param  {Object} options
+ *  @property {Function} options.log a method for writing log messages
+ *  @property {string} options.indexName name of the index in elasticsearch
+ *  @property {Function} options.callCluster a function for executing client requests
+ *  @property {Array<Object>} options.types an array of objects with `name` and `mapping` properties
+ *                                        describing the types that should be in the index
+ *  @return {Promise<undefined>}
+ */
+export async function ensureTypesExist({ log, indexName, callCluster, types }) {
+  const index = await callCluster('indices.get', {
+    index: indexName,
+    feature: '_mappings'
+  });
+
+  // could be different if aliases were resolved by `indices.get`
+  const resolvedName = Object.keys(index)[0];
+  const mappings = index[resolvedName].mappings;
+  const literalTypes = Object.keys(mappings);
+  const v6Index = literalTypes.length === 1 && literalTypes[0] === 'doc';
+
+  // our types aren't really es types, at least not in v6
+  const typesDefined = Object.keys(
+    v6Index
+      ? mappings.doc.properties
+      : mappings
+  );
+
+  for (const type of types) {
+    if (v6Index && type.name === '_default_') {
+      // v6 indices don't get _default_ types
+      continue;
+    }
+
+    const defined = typesDefined.includes(type.name);
+    if (defined) {
+      continue;
+    }
+
+    log(['info', 'elasticsearch'], {
+      tmpl: `Adding mappings to kibana index for SavedObject type "<%= typeName %>"`,
+      typeName: type.name,
+      typeMapping: type.mapping
+    });
+
+    if (v6Index) {
+      await callCluster('indices.putMapping', {
+        index: indexName,
+        type: 'doc',
+        body: {
+          properties: {
+            [type.name]: type.mapping
+          }
+        }
+      });
+    } else {
+      await callCluster('indices.putMapping', {
+        index: indexName,
+        type: type.name,
+        body: type.mapping
+      });
+    }
+  }
+}
diff --git a/src/core_plugins/elasticsearch/lib/health_check.js b/src/core_plugins/elasticsearch/lib/health_check.js
@@ -8,6 +8,7 @@ import { ensureEsVersion } from './ensure_es_version';
 import { ensureNotTribe } from './ensure_not_tribe';
 import { ensureAllowExplicitIndex } from './ensure_allow_explicit_index';
 import { determineEnabledScriptingLangs } from './determine_enabled_scripting_langs';
+import { ensureTypesExist } from './ensure_types_exist';
 
 const NoConnections = elasticsearch.errors.NoConnections;
 import util from 'util';
@@ -101,6 +102,12 @@ module.exports = function (plugin, server, { mappings }) {
       .then(() => ensureNotTribe(callAdminAsKibanaUser))
       .then(() => ensureAllowExplicitIndex(callAdminAsKibanaUser, config))
       .then(waitForShards)
+      .then(() => ensureTypesExist({
+        callCluster: callAdminAsKibanaUser,
+        log: (...args) => server.log(...args),
+        indexName: config.get('kibana.index'),
+        types: Object.keys(mappings).map(name => ({ name, mapping: mappings[name] }))
+      }))
       .then(_.partial(migrateConfig, server, { mappings }))
       .then(async () => {
         results.enabledScriptingLangs = await determineEnabledScriptingLangs(callDataAsKibanaUser);

diff --git a/test/functional/apps/dashboard/_dashboard_clone.js b/test/functional/apps/dashboard/_dashboard_clone.js
@@ -4,7 +4,7 @@ export default function ({ getService, getPageObjects }) {
   const retry = getService('retry');
   const PageObjects = getPageObjects(['dashboard', 'header', 'common']);
 
-  describe('dashboard save', function describeIndexTests() {
+  describe('dashboard clone', function describeIndexTests() {
     const dashboardName = 'Dashboard Clone Test';
     const clonedDashboardName = dashboardName + ' Copy';