[5.x] Disable dynamic/Implement static mappings (#10638) (#11417)

* Disable dynamic/Implement static mappings (#10638)

* Disable dynamic mappings

* Disable automatic type creation

* Centralize .kibana mappings

* Use dynamic strict instead of false

* Register mappings per plugin

* Check for registered mapping conflicts

* Expose mappings on server scope instead of module

* Stub elasticsearch mappings class

* Add tests for KibanaMappings class

* [mappings tests] to.an -> to.be.an

* Return on mapping conflict and include plugin id if provided

* [uiExports] Add mappings uiExport type

* Use uiExports config in mappings tests

* Clean up ui_mappings test

* Revert plugin require ordering

* Pull uiExports mappings from kibana object isntead of kbnServer

* [test] Update index mapping to look for keyword

src/core_plugins/elasticsearch/index.js

@@ -13,10 +13,9 @@ import createProxy, { createPath } from './lib/create_proxy';
 
 const DEFAULT_REQUEST_HEADERS = [ 'authorization' ];
 
-module.exports = function ({ Plugin }) {
-  return new Plugin({
+module.exports = function (kibana) {
+  return new kibana.Plugin({
     require: ['kibana'],
-
     config(Joi) {
       const { array, boolean, number, object, string, ref } = Joi;
 
@@ -159,9 +158,9 @@ module.exports = function ({ Plugin }) {
           pre: [ noDirectIndex, noBulkCheck ]
         }
       );
-
       // Set up the health check service and start it.
-      const { start, waitUntilReady } = healthCheck(this, server);
+      const mappings = kibana.uiExports.mappings.getCombined();
+      const { start, waitUntilReady } = healthCheck(this, server, { mappings });
       server.expose('waitUntilReady', waitUntilReady);
       start();
     }

src/core_plugins/elasticsearch/lib/__tests__/create_kibana_index.js

@@ -2,7 +2,7 @@ import _ from 'lodash';
 import sinon from 'sinon';
 import expect from 'expect.js';
 import Promise from 'bluebird';
-
+import mappings from './fixtures/mappings';
 import createKibanaIndex from '../create_kibana_index';
 import SetupError from '../setup_error';
 
@@ -23,13 +23,12 @@ describe('plugins/elasticsearch', function () {
       get.withArgs('kibana.index').returns(config.kibana.index);
       config = function () { return { get: get }; };
 
-      _.set(server, 'plugins.elasticsearch', {});
       _.set(server, 'config', config);
 
       callWithInternalUser = sinon.stub();
       cluster = { callWithInternalUser: callWithInternalUser };
 
-      server.plugins.elasticsearch.getCluster = sinon.stub().withArgs('admin').returns(cluster);
+      _.set(server, 'plugins.elasticsearch.getCluster', sinon.stub().withArgs('admin').returns(cluster));
     });
 
     describe('successful requests', function () {
@@ -39,14 +38,14 @@ describe('plugins/elasticsearch', function () {
       });
 
       it('should check cluster.health upon successful index creation', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           sinon.assert.calledOnce(callWithInternalUser.withArgs('cluster.health', sinon.match.any));
         });
       });
 
       it('should be created with mappings for config.buildNum', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)
@@ -60,14 +59,12 @@ describe('plugins/elasticsearch', function () {
           expect(params.body.mappings.config.properties)
             .to.have.property('buildNum');
           expect(params.body.mappings.config.properties.buildNum)
-            .to.have.property('type', 'string');
-          expect(params.body.mappings.config.properties.buildNum)
-            .to.have.property('index', 'not_analyzed');
+            .to.have.property('type', 'keyword');
         });
       });
 
       it('should be created with 1 shard and default replica', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)
@@ -82,7 +79,7 @@ describe('plugins/elasticsearch', function () {
       });
 
       it('should be created with index name set in the config', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)

src/core_plugins/elasticsearch/lib/__tests__/fixtures/mappings.js

@@ -0,0 +1,13 @@
+module.exports = {
+  '_default_': {
+    'dynamic': 'strict'
+  },
+  config: {
+    dynamic: true,
+    properties: {
+      buildNum: {
+        type: 'keyword'
+      }
+    }
+  }
+};

src/core_plugins/elasticsearch/lib/__tests__/health_check.js

@@ -5,6 +5,7 @@ import url from 'url';
 
 const NoConnections = require('elasticsearch').errors.NoConnections;
 
+import mappings from './fixtures/mappings';
 import healthCheck from '../health_check';
 import kibanaVersion from '../kibana_version';
 import serverConfig from '../../../../../test/server_config';
@@ -70,7 +71,7 @@ describe('plugins/elasticsearch', () => {
         }
       };
 
-      health = healthCheck(plugin, server);
+      health = healthCheck(plugin, server, { mappings });
     });
 
     afterEach(() => {

src/core_plugins/elasticsearch/lib/create_kibana_index.js

@@ -1,7 +1,6 @@
 import SetupError from './setup_error';
-import { mappings } from './kibana_index_mappings';
 
-module.exports = function (server) {
+module.exports = function (server, mappings) {
   const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
   const index = server.config().get('kibana.index');
 
@@ -15,7 +14,8 @@ module.exports = function (server) {
     index: index,
     body: {
       settings: {
-        number_of_shards: 1
+        number_of_shards: 1,
+        'index.mapper.dynamic': false,
       },
       mappings
     }

src/core_plugins/elasticsearch/lib/health_check.js

@@ -16,7 +16,7 @@ const NO_INDEX = 'no_index';
 const INITIALIZING = 'initializing';
 const READY = 'ready';
 
-module.exports = function (plugin, server) {
+module.exports = function (plugin, server, { mappings }) {
   const config = server.config();
   const callAdminAsKibanaUser = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser;
   const callDataAsKibanaUser = server.plugins.elasticsearch.getCluster('data').callWithInternalUser;
@@ -70,7 +70,7 @@ module.exports = function (plugin, server) {
     .then(function (health) {
       if (health === NO_INDEX) {
         plugin.status.yellow('No existing Kibana index found');
-        return createKibanaIndex(server);
+        return createKibanaIndex(server, mappings);
       }
 
       if (health === INITIALIZING) {
@@ -98,7 +98,7 @@ module.exports = function (plugin, server) {
       .then(() => ensureNotTribe(callAdminAsKibanaUser))
       .then(() => ensureAllowExplicitIndex(callAdminAsKibanaUser, config))
       .then(waitForShards)
-      .then(_.partial(migrateConfig, server))
+      .then(_.partial(migrateConfig, server, { mappings }))
       .then(() => {
         const tribeUrl = config.get('elasticsearch.tribe.url');
         if (tribeUrl) {

src/core_plugins/elasticsearch/lib/migrate_config.js

@@ -1,10 +1,9 @@
+import { get } from 'lodash';
 import upgrade from './upgrade_config';
-import { mappings } from './kibana_index_mappings';
 
-module.exports = function (server) {
+module.exports = function (server, { mappings }) {
   const config = server.config();
   const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
-
   const options =  {
     index: config.get('kibana.index'),
     type: 'config',
@@ -14,7 +13,7 @@ module.exports = function (server) {
         {
           buildNum: {
             order: 'desc',
-            unmapped_type: mappings.config.properties.buildNum.type
+            unmapped_type: get(mappings, 'config.properties.buildNum.type') || 'keyword'
           }
         }
       ]

src/core_plugins/kibana/index.js

@@ -9,14 +9,14 @@ import search from './server/routes/api/search';
 import settings from './server/routes/api/settings';
 import scripts from './server/routes/api/scripts';
 import * as systemApi from './server/lib/system_api';
+import mappings from './mappings.json';
 
 const mkdirp = Promise.promisify(mkdirpNode);
 
 module.exports = function (kibana) {
   const kbnBaseUrl = '/app/kibana';
   return new kibana.Plugin({
     id: 'kibana',
-
     config: function (Joi) {
       return Joi.object({
         enabled: Joi.boolean().default(true),
@@ -42,7 +42,6 @@ module.exports = function (kibana) {
           'devTools',
           'docViews'
         ],
-
         injectVars: function (server) {
           const serverConfig = server.config();
 
@@ -120,7 +119,8 @@ module.exports = function (kibana) {
 
       translations: [
         resolve(__dirname, './translations/en.json')
-      ]
+      ],
+      mappings
     },
 
     preInit: async function (server) {