From 374ac9ecd35c80a6ecd7b681ffadc17eb391d348 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 21 Nov 2019 14:27:59 +0100 Subject: [PATCH 01/19] Convert parts of elasticsearch version check to ts --- .../elasticsearch/lib/ensure_es_version.js | 2 +- ...js => is_es_compatible_with_kibana.test.ts} | 18 ++++++++---------- ...bana.js => is_es_compatible_with_kibana.ts} | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) rename src/legacy/core_plugins/elasticsearch/lib/{__tests__/is_es_compatible_with_kibana.js => is_es_compatible_with_kibana.test.ts} (72%) rename src/legacy/core_plugins/elasticsearch/lib/{is_es_compatible_with_kibana.js => is_es_compatible_with_kibana.ts} (95%) diff --git a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js index 8d304cd558418..eefd34af47abd 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js +++ b/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js @@ -24,7 +24,7 @@ import { forEach, get } from 'lodash'; import { coerce } from 'semver'; -import isEsCompatibleWithKibana from './is_es_compatible_with_kibana'; +import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; /** * tracks the node descriptions that get logged in warnings so diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js b/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.test.ts similarity index 72% rename from src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js rename to src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.test.ts index 092c0ecf1071c..374ec240b2f49 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js +++ b/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.test.ts @@ -17,41 +17,39 @@ * under the License. */ -import expect from '@kbn/expect'; - -import isEsCompatibleWithKibana from '../is_es_compatible_with_kibana'; +import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; describe('plugins/elasticsearch', () => { describe('lib/is_es_compatible_with_kibana', () => { describe('returns false', () => { it('when ES major is greater than Kibana major', () => { - expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).to.be(false); + expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).toBe(false); }); it('when ES major is less than Kibana major', () => { - expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).to.be(false); + expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).toBe(false); }); it('when majors are equal, but ES minor is less than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).to.be(false); + expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).toBe(false); }); }); describe('returns true', () => { it('when version numbers are the same', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).to.be(true); + expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).toBe(true); }); it('when majors are equal, and ES minor is greater than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).to.be(true); + expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).toBe(true); }); it('when majors and minors are equal, and ES patch is greater than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).to.be(true); + expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).toBe(true); }); it('when majors and minors are equal, but ES patch is less than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).to.be(true); + expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).toBe(true); }); }); }); diff --git a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js b/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.ts similarity index 95% rename from src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js rename to src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.ts index 4afbd488d2946..fd0bdd29ff837 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js +++ b/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.ts @@ -25,7 +25,7 @@ import semver from 'semver'; -export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) { +export function isEsCompatibleWithKibana(esVersion: string, kibanaVersion: string) { const esVersionNumbers = { major: semver.major(esVersion), minor: semver.minor(esVersion), From e2d6157a4fc5707f86787a191b5e84702cd02475 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 21 Nov 2019 17:47:31 +0100 Subject: [PATCH 02/19] Move ES version check to NP --- .../elasticsearch/elasticsearch_config.ts | 14 +- .../elasticsearch_service.mock.ts | 3 +- .../elasticsearch/elasticsearch_service.ts | 39 ++- src/core/server/elasticsearch/types.ts | 1 + .../version_check/ensure_es_version.test.ts | 177 ++++++++++++++ .../version_check/ensure_es_version.ts | 141 +++++++++++ .../is_es_compatible_with_kibana.test.ts | 0 .../is_es_compatible_with_kibana.ts | 0 .../core_plugins/elasticsearch/index.js | 14 +- .../lib/__tests__/ensure_es_version.js | 223 ------------------ .../lib/__tests__/health_check.js | 151 ------------ .../elasticsearch/lib/ensure_es_version.js | 126 ---------- .../elasticsearch/lib/health_check.js | 75 ------ ...ana_version.js => version_health_check.js} | 24 +- .../lib/version_health_check.test.js | 71 ++++++ 15 files changed, 460 insertions(+), 599 deletions(-) create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.test.ts create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.ts rename src/{legacy/core_plugins/elasticsearch/lib => core/server/elasticsearch/version_check}/is_es_compatible_with_kibana.test.ts (100%) rename src/{legacy/core_plugins/elasticsearch/lib => core/server/elasticsearch/version_check}/is_es_compatible_with_kibana.ts (100%) delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/health_check.js rename src/legacy/core_plugins/elasticsearch/lib/{kibana_version.js => version_health_check.js} (56%) create mode 100644 src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js diff --git a/src/core/server/elasticsearch/elasticsearch_config.ts b/src/core/server/elasticsearch/elasticsearch_config.ts index 5f06c51a53d53..c0fe3fd172b20 100644 --- a/src/core/server/elasticsearch/elasticsearch_config.ts +++ b/src/core/server/elasticsearch/elasticsearch_config.ts @@ -103,7 +103,19 @@ const configSchema = schema.object({ ), apiVersion: schema.string({ defaultValue: DEFAULT_API_VERSION }), healthCheck: schema.object({ delay: schema.duration({ defaultValue: 2500 }) }), - ignoreVersionMismatch: schema.boolean({ defaultValue: false }), + ignoreVersionMismatch: schema.conditional( + schema.contextRef('dev'), + false, + schema.any({ + validate: rawValue => { + if (rawValue === true) { + return '"ignoreVersionMismatch" can only be set to true in development mode'; + } + }, + defaultValue: false, + }), + schema.boolean({ defaultValue: false }) + ), }); const deprecations: ConfigDeprecationProvider = () => [ diff --git a/src/core/server/elasticsearch/elasticsearch_service.mock.ts b/src/core/server/elasticsearch/elasticsearch_service.mock.ts index a4e51ca55b3e7..f413235edfbd0 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.mock.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.mock.ts @@ -17,7 +17,7 @@ * under the License. */ -import { BehaviorSubject } from 'rxjs'; +import { BehaviorSubject, Subject } from 'rxjs'; import { IClusterClient, ICustomClusterClient } from './cluster_client'; import { IScopedClusterClient } from './scoped_cluster_client'; import { ElasticsearchConfig } from './elasticsearch_config'; @@ -71,6 +71,7 @@ type MockedInternalElasticSearchServiceSetup = jest.Mocked< const createInternalSetupContractMock = () => { const setupContract: MockedInternalElasticSearchServiceSetup = { ...createSetupContractMock(), + esNodesCompatibility$: new Subject(), legacy: { config$: new BehaviorSubject({} as ElasticsearchConfig), }, diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts index aba246ce66fb5..44d261a03b2c9 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.ts @@ -30,6 +30,7 @@ import { ElasticsearchConfig, ElasticsearchConfigType } from './elasticsearch_co import { InternalHttpServiceSetup, GetAuthHeaders } from '../http/'; import { InternalElasticsearchServiceSetup } from './types'; import { CallAPIOptions } from './api_types'; +import { pollEsNodesVersion } from './version_check/ensure_es_version'; /** @internal */ interface CoreClusterClients { @@ -46,9 +47,17 @@ interface SetupDeps { export class ElasticsearchService implements CoreService { private readonly log: Logger; private readonly config$: Observable; - private subscription?: Subscription; + private subscriptions: { + client?: Subscription; + esNodesCompatibility?: Subscription; + } = { + client: undefined, + esNodesCompatibility: undefined, + }; + private kibanaVersion: string; constructor(private readonly coreContext: CoreContext) { + this.kibanaVersion = coreContext.env.packageInfo.version; this.log = coreContext.logger.get('elasticsearch-service'); this.config$ = coreContext.configService .atPath('elasticsearch') @@ -60,7 +69,7 @@ export class ElasticsearchService implements CoreService { - if (this.subscription !== undefined) { + if (this.subscriptions.client !== undefined) { this.log.error('Clients cannot be changed after they are created'); return false; } @@ -91,7 +100,7 @@ export class ElasticsearchService implements CoreService; - this.subscription = clients$.connect(); + this.subscriptions.client = clients$.connect(); const config = await this.config$.pipe(first()).toPromise(); @@ -149,11 +158,24 @@ export class ElasticsearchService implements CoreService).connect(); + return { legacy: { config$: clients$.pipe(map(clients => clients.config)) }, adminClient, dataClient, + esNodesCompatibility$, createClient: (type: string, clientConfig: Partial = {}) => { const finalConfig = merge({}, config, clientConfig); @@ -166,11 +188,12 @@ export class ElasticsearchService implements CoreService; }; + esNodesCompatibility$: Observable; } diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts new file mode 100644 index 0000000000000..4c87773d2fcf8 --- /dev/null +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts @@ -0,0 +1,177 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { mapNodesVersionCompatibility, pollEsNodesVersion, NodesInfo } from './ensure_es_version'; +import { loggingServiceMock } from '../../logging/logging_service.mock'; +import { take } from 'rxjs/operators'; + +const mockLoggerFactory = loggingServiceMock.create(); +const mockLogger = mockLoggerFactory.get('mock logger'); + +const KIBANA_VERSION = '5.1.0'; + +function createNodes(...versions: string[]): NodesInfo { + const nodes = {} as any; + versions + .map(version => { + return { + version, + http: { + publish_address: 'http_address', + }, + ip: 'ip', + }; + }) + .forEach((node, i) => { + nodes[`node-${i}`] = node; + }); + + return { nodes }; +} + +describe('mapNodesVersionCompatibility', () => { + function createNodesInfoWithoutHTTP(version: string): NodesInfo { + return { nodes: { 'node-without-http': { version, ip: 'ip' } } } as any; + } + + it('returns isCompatible=true with a single node that matches', async () => { + const nodesInfo = createNodes('5.1.0'); + const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + }); + + it('returns isCompatible=true with multiple nodes that satisfy', async () => { + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.1.1-Beta1'); + const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + }); + + it('returns isCompatible=false for a single node that is out of date', () => { + // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0'); + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(false); + expect(result.message).toMatchInlineSnapshot( + `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v5.0.0 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=false for an incompatible node without http publish address', async () => { + const nodesInfo = createNodesInfoWithoutHTTP('6.1.1'); + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(false); + expect(result.message).toMatchInlineSnapshot( + `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v6.1.1 @ undefined (ip)"` + ); + }); + + it('returns isCompatible=true for outdated nodes when ignoreVersionMismatch=true', async () => { + // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0'); + const ignoreVersionMismatch = true; + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, ignoreVersionMismatch); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"Ignoring version incompatibility between Kibana v5.1.0 and the following Elasticsearch nodes: v5.0.0 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=true with a message if a node is only off by a patch version', () => { + const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=true with a message if a node is only off by a patch version and without http publish address', async () => { + const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"` + ); + }); +}); + +describe('pollEsNodesVersion', () => { + const callWithInternalUser = jest.fn(); + it('keeps polling when a poll request throws', done => { + expect.assertions(2); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); + callWithInternalUser.mockRejectedValueOnce(new Error('mock request error')); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.1.1-Beta1')); + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(2)) + .subscribe({ + next: result => expect(result.isCompatible).toBeDefined(), + complete: done, + error: done, + }); + }); + + it('returns compatibility results', done => { + expect.assertions(1); + const nodes = createNodes('5.1.0', '5.2.0', '5.0.0'); + callWithInternalUser.mockResolvedValueOnce(nodes); + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(1)) + .subscribe({ + next: result => { + expect(result).toEqual(mapNodesVersionCompatibility(nodes, KIBANA_VERSION, false)); + }, + complete: done, + error: done, + }); + }); + + it('only emits if the node versions changed since the previous poll', done => { + expect.assertions(4); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // ignore, same versions, different ordering + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.2.0', '5.0.0')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // ignore + callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // emit, different from previous version + + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(4)) + .subscribe({ + next: result => expect(result).toBeDefined(), + complete: done, + error: done, + }); + }); +}); diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts new file mode 100644 index 0000000000000..d04f1bb9a879a --- /dev/null +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * ES and Kibana versions are locked, so Kibana should require that ES has the same version as + * that defined in Kibana's package.json. + */ + +import { coerce } from 'semver'; +import { interval } from 'rxjs'; +import { map, switchMap, catchError, distinctUntilChanged } from 'rxjs/operators'; +import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; +import { Logger } from '../../logging'; +import { APICaller } from '..'; + +export interface EnsureVersionOptions { + callWithInternalUser: APICaller; + log: Logger; + kibanaVersion: string; + ignoreVersionMismatch: boolean; + esVersionCheckInterval: number; +} + +export interface NodesInfo { + nodes: { + [key: string]: NodeInfo; + }; +} + +interface NodeInfo { + version: string; + ip: string; + http: { + publish_address: string; + }; +} + +function getHumanizedNodeName(node: NodeInfo) { + const publishAddress = node?.http?.publish_address + ' ' || ''; + return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')'; +} + +export function mapNodesVersionCompatibility( + nodesInfo: NodesInfo, + kibanaVersion: string, + ignoreVersionMismatch: boolean +) { + const nodes = Object.keys(nodesInfo.nodes) + .sort() // Sorting ensures a stable node ordering for comparison + .map(key => nodesInfo.nodes[key]) + .map(node => Object.assign({}, node, { name: getHumanizedNodeName(node) })); + + // Aggregate incompatible ES nodes. + const incompatibleNodes = nodes.filter( + node => !isEsCompatibleWithKibana(node.version, kibanaVersion) + ); + + // Aggregate ES nodes which should prompt a Kibana upgrade. It's acceptable + // if ES and Kibana versions are not the same so long as they are not + // incompatible, but we should warn about it. + // Ignore version qualifiers https://github.com/elastic/elasticsearch/issues/36859 + const warningNodes = nodes.filter(node => { + const nodeSemVer = coerce(node.version); + const kibanaSemver = coerce(kibanaVersion); + return nodeSemVer && kibanaSemver && nodeSemVer.version !== kibanaSemver.version; + }); + + let message = {}; + if (incompatibleNodes.length > 0) { + const incompatibleNodeNames = incompatibleNodes.map(node => node.name).join(', '); + if (ignoreVersionMismatch) { + message = `Ignoring version incompatibility between Kibana v${kibanaVersion} and the following Elasticsearch nodes: ${incompatibleNodeNames}`; + } else { + message = `This version of Kibana (v${kibanaVersion}) is incompatible with the following Elasticsearch nodes in your cluster: ${incompatibleNodeNames}`; + } + } else if (warningNodes.length > 0) { + const warningNodeNames = warningNodes.map(node => node.name).join(', '); + message = + `You're running Kibana ${kibanaVersion} with some different versions of ` + + 'Elasticsearch. Update Kibana or Elasticsearch to the same ' + + `version to prevent compatibility issues: ${warningNodeNames}`; + } + + return { + isCompatible: ignoreVersionMismatch || incompatibleNodes.length === 0, + message, + incompatibleNodes, + warningNodes, + kibanaVersion, + }; +} + +export const pollEsNodesVersion = ({ + callWithInternalUser, + log, + kibanaVersion, + ignoreVersionMismatch, + esVersionCheckInterval: healthCheckInterval, +}: EnsureVersionOptions) => { + log.debug('Checking Elasticsearch version'); + + return interval(healthCheckInterval).pipe( + switchMap(() => { + return callWithInternalUser('nodes.info', { + filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], + }); + }), + // Log, but otherwise ignore 'nodes.info' request errors + catchError((err, caught$) => { + log.error('Unable to retrieve version information from Elasticsearch nodes.', err); + return caught$; + }), + map((nodesInfo: NodesInfo) => + mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch) + ), + // Only emit if the IP or version numbers of the nodes + distinctUntilChanged((prev, curr) => { + const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version; + return ( + curr.incompatibleNodes.every((node, i) => nodesEqual(node, prev.incompatibleNodes[i])) && + curr.warningNodes.every((node, i) => nodesEqual(node, prev.warningNodes[i])) + ); + }) + ); +}; diff --git a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.test.ts b/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.test.ts similarity index 100% rename from src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.test.ts rename to src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.test.ts diff --git a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.ts b/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.ts similarity index 100% rename from src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.ts rename to src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.ts diff --git a/src/legacy/core_plugins/elasticsearch/index.js b/src/legacy/core_plugins/elasticsearch/index.js index da7b557e7ea19..773d48f42917d 100644 --- a/src/legacy/core_plugins/elasticsearch/index.js +++ b/src/legacy/core_plugins/elasticsearch/index.js @@ -17,10 +17,10 @@ * under the License. */ import { first } from 'rxjs/operators'; -import healthCheck from './lib/health_check'; import { Cluster } from './lib/cluster'; import { createProxy } from './lib/create_proxy'; import { handleESError } from './lib/handle_es_error'; +import { versionHealthCheck } from './lib/version_health_check'; export default function(kibana) { let defaultVars; @@ -92,15 +92,13 @@ export default function(kibana) { createProxy(server); - // Set up the health check service and start it. - const { start, waitUntilReady } = healthCheck( + const waitUntilHealthy = versionHealthCheck( this, - server, - esConfig.healthCheckDelay.asMilliseconds(), - esConfig.ignoreVersionMismatch + server.logWithMetadata, + server.newPlatform.__internals.elasticsearch.esNodesCompatibilty$ ); - server.expose('waitUntilReady', waitUntilReady); - start(); + + server.expose('waitUntilReady', () => waitUntilHealthy); }, }); } diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js deleted file mode 100644 index 781d198c66236..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import _ from 'lodash'; -import Bluebird from 'bluebird'; -import sinon from 'sinon'; -import expect from '@kbn/expect'; - -import { esTestConfig } from '@kbn/test'; -import { ensureEsVersion } from '../ensure_es_version'; - -describe('plugins/elasticsearch', () => { - describe('lib/ensure_es_version', () => { - const KIBANA_VERSION = '5.1.0'; - - let server; - - beforeEach(function() { - server = { - log: sinon.stub(), - logWithMetadata: sinon.stub(), - plugins: { - elasticsearch: { - getCluster: sinon - .stub() - .withArgs('admin') - .returns({ callWithInternalUser: sinon.stub() }), - status: { - red: sinon.stub(), - }, - url: esTestConfig.getUrl(), - }, - }, - config() { - return { - get: sinon.stub(), - }; - }, - }; - }); - - function setNodes(/* ...versions */) { - const versions = _.shuffle(arguments); - const nodes = {}; - let i = 0; - - while (versions.length) { - const name = 'node-' + ++i; - const version = versions.shift(); - - const node = { - version: version, - http: { - publish_address: 'http_address', - }, - ip: 'ip', - }; - - if (!_.isString(version)) _.assign(node, version); - nodes[name] = node; - } - - const cluster = server.plugins.elasticsearch.getCluster('admin'); - cluster.callWithInternalUser - .withArgs('nodes.info', sinon.match.any) - .returns(Bluebird.resolve({ nodes: nodes })); - } - - function setNodeWithoutHTTP(version) { - const nodes = { 'node-without-http': { version, ip: 'ip' } }; - const cluster = server.plugins.elasticsearch.getCluster('admin'); - cluster.callWithInternalUser - .withArgs('nodes.info', sinon.match.any) - .returns(Bluebird.resolve({ nodes: nodes })); - } - - it('returns true with single a node that matches', async () => { - setNodes('5.1.0'); - const result = await ensureEsVersion(server, KIBANA_VERSION); - expect(result).to.be(true); - }); - - it('returns true with multiple nodes that satisfy', async () => { - setNodes('5.1.0', '5.2.0', '5.1.1-Beta1'); - const result = await ensureEsVersion(server, KIBANA_VERSION); - expect(result).to.be(true); - }); - - it('throws an error with a single node that is out of date', async () => { - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('does not throw on outdated nodes, if `ignoreVersionMismatch` is enabled in development mode', async () => { - // set config values - server.config = () => ({ - get: name => { - switch (name) { - case 'env.dev': - return true; - default: - throw new Error(`Unknown option "${name}"`); - } - }, - }); - - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - - const ignoreVersionMismatch = true; - const result = await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch); - expect(result).to.be(true); - }); - - it('throws an error if `ignoreVersionMismatch` is enabled in production mode', async () => { - // set config values - server.config = () => ({ - get: name => { - switch (name) { - case 'env.dev': - return false; - default: - throw new Error(`Unknown option "${name}"`); - } - }, - }); - - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - - try { - const ignoreVersionMismatch = true; - await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('fails if that single node is a client node', async () => { - setNodes('5.1.0', '5.2.0', { version: '5.0.0', attributes: { client: 'true' } }); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('warns if a node is only off by a patch version', async () => { - setNodes('5.1.1'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - }); - - it('warns if a node is off by a patch version and without http publish address', async () => { - setNodeWithoutHTTP('5.1.1'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - }); - - it('errors if a node incompatible and without http publish address', async () => { - setNodeWithoutHTTP('6.1.1'); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e.message).to.contain('incompatible nodes'); - expect(e).to.be.a(Error); - } - }); - - it('only warns once per node list', async () => { - setNodes('5.1.1'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 3); - expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug'); - }); - - it('warns again if the node list changes', async () => { - setNodes('5.1.1'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - - setNodes('5.1.2'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 4); - expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(3).args[0]).to.contain('warning'); - }); - }); -}); diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js deleted file mode 100644 index 3b593c6352394..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Bluebird from 'bluebird'; -import sinon from 'sinon'; -import expect from '@kbn/expect'; - -const NoConnections = require('elasticsearch').errors.NoConnections; - -import healthCheck from '../health_check'; -import kibanaVersion from '../kibana_version'; - -const esPort = 9220; - -describe('plugins/elasticsearch', () => { - describe('lib/health_check', function() { - let health; - let plugin; - let cluster; - let server; - const sandbox = sinon.createSandbox(); - - function getTimerCount() { - return Object.keys(sandbox.clock.timers || {}).length; - } - - beforeEach(() => { - sandbox.useFakeTimers(); - const COMPATIBLE_VERSION_NUMBER = '5.0.0'; - - // Stub the Kibana version instead of drawing from package.json. - sandbox.stub(kibanaVersion, 'get').returns(COMPATIBLE_VERSION_NUMBER); - - // setup the plugin stub - plugin = { - name: 'elasticsearch', - status: { - red: sinon.stub(), - green: sinon.stub(), - yellow: sinon.stub(), - }, - }; - - cluster = { callWithInternalUser: sinon.stub(), errors: { NoConnections } }; - cluster.callWithInternalUser.withArgs('index', sinon.match.any).returns(Bluebird.resolve()); - cluster.callWithInternalUser - .withArgs('mget', sinon.match.any) - .returns(Bluebird.resolve({ ok: true })); - cluster.callWithInternalUser - .withArgs('get', sinon.match.any) - .returns(Bluebird.resolve({ found: false })); - cluster.callWithInternalUser - .withArgs('search', sinon.match.any) - .returns(Bluebird.resolve({ hits: { hits: [] } })); - cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any).returns( - Bluebird.resolve({ - nodes: { - 'node-01': { - version: COMPATIBLE_VERSION_NUMBER, - http_address: `inet[/127.0.0.1:${esPort}]`, - ip: '127.0.0.1', - }, - }, - }) - ); - - // Setup the server mock - server = { - logWithMetadata: sinon.stub(), - info: { port: 5601 }, - config: () => ({ get: sinon.stub() }), - plugins: { - elasticsearch: { - getCluster: sinon.stub().returns(cluster), - }, - }, - ext: sinon.stub(), - }; - - health = healthCheck(plugin, server, 0); - }); - - afterEach(() => sandbox.restore()); - - it('should stop when cluster is shutdown', () => { - // ensure that health.start() is responsible for the timer we are observing - expect(getTimerCount()).to.be(0); - health.start(); - expect(getTimerCount()).to.be(1); - - // ensure that a server extension was registered - sinon.assert.calledOnce(server.ext); - sinon.assert.calledWithExactly(server.ext, sinon.match.string, sinon.match.func); - - const [, handler] = server.ext.firstCall.args; - handler(); // this should be health.stop - - // ensure that the handler unregistered the timer - expect(getTimerCount()).to.be(0); - }); - - it('should set the cluster green if everything is ready', function() { - cluster.callWithInternalUser.withArgs('ping').returns(Bluebird.resolve()); - - return health.run().then(function() { - sinon.assert.calledOnce(plugin.status.yellow); - sinon.assert.calledWithExactly(plugin.status.yellow, 'Waiting for Elasticsearch'); - - sinon.assert.calledOnce( - cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any) - ); - sinon.assert.notCalled(plugin.status.red); - sinon.assert.calledOnce(plugin.status.green); - sinon.assert.calledWithExactly(plugin.status.green, 'Ready'); - }); - }); - - describe('#waitUntilReady', function() { - it('waits for green status', function() { - plugin.status.once = sinon.spy(function(event, handler) { - expect(event).to.be('green'); - setImmediate(handler); - }); - - const waitUntilReadyPromise = health.waitUntilReady(); - - sandbox.clock.runAll(); - - return waitUntilReadyPromise.then(function() { - sinon.assert.calledOnce(plugin.status.once); - }); - }); - }); - }); -}); diff --git a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js deleted file mode 100644 index eefd34af47abd..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * ES and Kibana versions are locked, so Kibana should require that ES has the same version as - * that defined in Kibana's package.json. - */ - -import { forEach, get } from 'lodash'; -import { coerce } from 'semver'; -import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; - -/** - * tracks the node descriptions that get logged in warnings so - * that we don't spam the log with the same message over and over. - * - * There are situations, like in testing or multi-tenancy, where - * the server argument changes, so we must track the previous - * node warnings per server - */ -const lastWarnedNodesForServer = new WeakMap(); - -export function ensureEsVersion(server, kibanaVersion, ignoreVersionMismatch = false) { - const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin'); - - server.logWithMetadata(['plugin', 'debug'], 'Checking Elasticsearch version'); - return callWithInternalUser('nodes.info', { - filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], - }).then(function(info) { - // Aggregate incompatible ES nodes. - const incompatibleNodes = []; - - // Aggregate ES nodes which should prompt a Kibana upgrade. - const warningNodes = []; - - forEach(info.nodes, esNode => { - if (!isEsCompatibleWithKibana(esNode.version, kibanaVersion)) { - // Exit early to avoid collecting ES nodes with newer major versions in the `warningNodes`. - return incompatibleNodes.push(esNode); - } - - // It's acceptable if ES and Kibana versions are not the same so long as - // they are not incompatible, but we should warn about it - - // Ignore version qualifiers - // https://github.com/elastic/elasticsearch/issues/36859 - const looseMismatch = coerce(esNode.version).version !== coerce(kibanaVersion).version; - if (looseMismatch) { - warningNodes.push(esNode); - } - }); - - function getHumanizedNodeNames(nodes) { - return nodes.map(node => { - const publishAddress = get(node, 'http.publish_address') - ? get(node, 'http.publish_address') + ' ' - : ''; - return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')'; - }); - } - - if (warningNodes.length) { - const simplifiedNodes = warningNodes.map(node => ({ - version: node.version, - http: { - publish_address: get(node, 'http.publish_address'), - }, - ip: node.ip, - })); - - // Don't show the same warning over and over again. - const warningNodeNames = getHumanizedNodeNames(simplifiedNodes).join(', '); - if (lastWarnedNodesForServer.get(server) !== warningNodeNames) { - lastWarnedNodesForServer.set(server, warningNodeNames); - server.logWithMetadata( - ['warning'], - `You're running Kibana ${kibanaVersion} with some different versions of ` + - 'Elasticsearch. Update Kibana or Elasticsearch to the same ' + - `version to prevent compatibility issues: ${warningNodeNames}`, - { - kibanaVersion, - nodes: simplifiedNodes, - } - ); - } - } - - if (incompatibleNodes.length && !shouldIgnoreVersionMismatch(server, ignoreVersionMismatch)) { - const incompatibleNodeNames = getHumanizedNodeNames(incompatibleNodes); - throw new Error( - `This version of Kibana requires Elasticsearch v` + - `${kibanaVersion} on all nodes. I found ` + - `the following incompatible nodes in your cluster: ${incompatibleNodeNames.join(', ')}` - ); - } - - return true; - }); -} - -function shouldIgnoreVersionMismatch(server, ignoreVersionMismatch) { - const isDevMode = server.config().get('env.dev'); - if (!isDevMode && ignoreVersionMismatch) { - throw new Error( - `Option "elasticsearch.ignoreVersionMismatch" can only be used in development mode` - ); - } - - return isDevMode && ignoreVersionMismatch; -} diff --git a/src/legacy/core_plugins/elasticsearch/lib/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/health_check.js deleted file mode 100644 index 40053ec774542..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/health_check.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Bluebird from 'bluebird'; -import kibanaVersion from './kibana_version'; -import { ensureEsVersion } from './ensure_es_version'; - -export default function(plugin, server, requestDelay, ignoreVersionMismatch) { - plugin.status.yellow('Waiting for Elasticsearch'); - - function waitUntilReady() { - return new Bluebird(resolve => { - plugin.status.once('green', resolve); - }); - } - - function check() { - return ensureEsVersion(server, kibanaVersion.get(), ignoreVersionMismatch) - .then(() => plugin.status.green('Ready')) - .catch(err => plugin.status.red(err)); - } - - let timeoutId = null; - - function scheduleCheck(ms) { - if (timeoutId) return; - - const myId = setTimeout(function() { - check().finally(function() { - if (timeoutId === myId) startorRestartChecking(); - }); - }, ms); - - timeoutId = myId; - } - - function startorRestartChecking() { - scheduleCheck(stopChecking() ? requestDelay : 1); - } - - function stopChecking() { - if (!timeoutId) return false; - clearTimeout(timeoutId); - timeoutId = null; - return true; - } - - server.ext('onPreStop', stopChecking); - - return { - waitUntilReady: waitUntilReady, - run: check, - start: startorRestartChecking, - stop: stopChecking, - isRunning: function() { - return !!timeoutId; - }, - }; -} diff --git a/src/legacy/core_plugins/elasticsearch/lib/kibana_version.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js similarity index 56% rename from src/legacy/core_plugins/elasticsearch/lib/kibana_version.js rename to src/legacy/core_plugins/elasticsearch/lib/version_health_check.js index e00c12f8271b0..5eaafccd7b8d9 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/kibana_version.js +++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js @@ -17,11 +17,23 @@ * under the License. */ -import { version as kibanaVersion } from '../../../../../package.json'; +export const versionHealthCheck = (esPlugin, logWithMetadata, esNodesCompatibility$) => { + esPlugin.status.yellow('Waiting for Elasticsearch'); -export default { - // Make the version stubbable to improve testability. - get() { - return kibanaVersion; - }, + return new Promise(resolve => { + esNodesCompatibility$.subscribe(({ isCompatible, message, kibanaVersion, warningNodes }) => { + if (!isCompatible) { + esPlugin.status.red(message); + } else { + if (message && message.length > 0) { + logWithMetadata(['warning'], message, { + kibanaVersion, + nodes: warningNodes, + }); + } + esPlugin.status.green('Ready'); + resolve(); + } + }); + }); }; diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js new file mode 100644 index 0000000000000..ba7c95bcdfec5 --- /dev/null +++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { versionHealthCheck } from './version_health_check'; +import { Subject } from 'rxjs'; + +describe('plugins/elasticsearch', () => { + describe('lib/health_version_check', function() { + let plugin; + let logWithMetadata; + + beforeEach(() => { + plugin = { + status: { + red: jest.fn(), + green: jest.fn(), + yellow: jest.fn(), + }, + }; + + logWithMetadata = jest.fn(); + jest.clearAllMocks(); + }); + + it('returned promise resolves when all nodes are compatible ', function() { + const esNodesCompatibility$ = new Subject(); + const versionHealthyPromise = versionHealthCheck( + plugin, + logWithMetadata, + esNodesCompatibility$ + ); + esNodesCompatibility$.next({ isCompatible: true, message: undefined }); + return expect(versionHealthyPromise).resolves.toBe(undefined); + }); + + it('should set elasticsearch plugin status to green when all nodes are compatible', function() { + const esNodesCompatibility$ = new Subject(); + versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$); + expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch'); + expect(plugin.status.green).not.toHaveBeenCalled(); + esNodesCompatibility$.next({ isCompatible: true, message: undefined }); + expect(plugin.status.green).toHaveBeenCalledWith('Ready'); + expect(plugin.status.red).not.toHaveBeenCalled(); + }); + + it('should set elasticsearch plugin status to red when some nodes are incompatible', function() { + const esNodesCompatibility$ = new Subject(); + versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$); + expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch'); + expect(plugin.status.red).not.toHaveBeenCalled(); + esNodesCompatibility$.next({ isCompatible: false, message: 'your nodes are incompatible' }); + expect(plugin.status.red).toHaveBeenCalledWith('your nodes are incompatible'); + expect(plugin.status.green).not.toHaveBeenCalled(); + }); + }); +}); From 573af7c1ba71a055b271c9febe80376c7e757656 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Fri, 24 Jan 2020 23:30:52 +0100 Subject: [PATCH 03/19] Improve types --- src/core/server/elasticsearch/types.ts | 3 ++- .../version_check/ensure_es_version.ts | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/core/server/elasticsearch/types.ts b/src/core/server/elasticsearch/types.ts index a959c0108aa62..90cfdcc035d8e 100644 --- a/src/core/server/elasticsearch/types.ts +++ b/src/core/server/elasticsearch/types.ts @@ -21,6 +21,7 @@ import { Observable } from 'rxjs'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchClientConfig } from './elasticsearch_client_config'; import { IClusterClient, ICustomClusterClient } from './cluster_client'; +import { NodesVersionCompatibility } from './version_check/ensure_es_version'; /** * @public @@ -77,5 +78,5 @@ export interface InternalElasticsearchServiceSetup extends ElasticsearchServiceS readonly legacy: { readonly config$: Observable; }; - esNodesCompatibility$: Observable; + esNodesCompatibility$: Observable; } diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index d04f1bb9a879a..238daa1760abc 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -29,7 +29,7 @@ import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; import { Logger } from '../../logging'; import { APICaller } from '..'; -export interface EnsureVersionOptions { +export interface PollEsNodesVersionOptions { callWithInternalUser: APICaller; log: Logger; kibanaVersion: string; @@ -49,6 +49,15 @@ interface NodeInfo { http: { publish_address: string; }; + name: string; +} + +export interface NodesVersionCompatibility { + isCompatible: boolean; + message?: string; + incompatibleNodes: NodeInfo[]; + warningNodes: NodeInfo[]; + kibanaVersion: string; } function getHumanizedNodeName(node: NodeInfo) { @@ -60,7 +69,7 @@ export function mapNodesVersionCompatibility( nodesInfo: NodesInfo, kibanaVersion: string, ignoreVersionMismatch: boolean -) { +): NodesVersionCompatibility { const nodes = Object.keys(nodesInfo.nodes) .sort() // Sorting ensures a stable node ordering for comparison .map(key => nodesInfo.nodes[key]) @@ -81,7 +90,7 @@ export function mapNodesVersionCompatibility( return nodeSemVer && kibanaSemver && nodeSemVer.version !== kibanaSemver.version; }); - let message = {}; + let message; if (incompatibleNodes.length > 0) { const incompatibleNodeNames = incompatibleNodes.map(node => node.name).join(', '); if (ignoreVersionMismatch) { @@ -112,7 +121,7 @@ export const pollEsNodesVersion = ({ kibanaVersion, ignoreVersionMismatch, esVersionCheckInterval: healthCheckInterval, -}: EnsureVersionOptions) => { +}: PollEsNodesVersionOptions) => { log.debug('Checking Elasticsearch version'); return interval(healthCheckInterval).pipe( @@ -129,7 +138,8 @@ export const pollEsNodesVersion = ({ map((nodesInfo: NodesInfo) => mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch) ), - // Only emit if the IP or version numbers of the nodes + // Only emit if the IP or version numbers of the nodes changed from the + // previous result. distinctUntilChanged((prev, curr) => { const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version; return ( From 91411fbf8b18a2fc158cfc5c82019d5ba47d1340 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Sun, 26 Jan 2020 21:26:38 +0100 Subject: [PATCH 04/19] Fix tests --- .../elasticsearch/elasticsearch_service.mock.ts | 10 ++++++++-- .../elasticsearch/elasticsearch_service.test.ts | 11 ++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/core/server/elasticsearch/elasticsearch_service.mock.ts b/src/core/server/elasticsearch/elasticsearch_service.mock.ts index f413235edfbd0..b8ad375496544 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.mock.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.mock.ts @@ -17,12 +17,13 @@ * under the License. */ -import { BehaviorSubject, Subject } from 'rxjs'; +import { BehaviorSubject } from 'rxjs'; import { IClusterClient, ICustomClusterClient } from './cluster_client'; import { IScopedClusterClient } from './scoped_cluster_client'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; import { InternalElasticsearchServiceSetup, ElasticsearchServiceSetup } from './types'; +import { NodesVersionCompatibility } from './version_check/ensure_es_version'; const createScopedClusterClientMock = (): jest.Mocked => ({ callAsInternalUser: jest.fn(), @@ -71,7 +72,12 @@ type MockedInternalElasticSearchServiceSetup = jest.Mocked< const createInternalSetupContractMock = () => { const setupContract: MockedInternalElasticSearchServiceSetup = { ...createSetupContractMock(), - esNodesCompatibility$: new Subject(), + esNodesCompatibility$: new BehaviorSubject({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }), legacy: { config$: new BehaviorSubject({} as ElasticsearchConfig), }, diff --git a/src/core/server/elasticsearch/elasticsearch_service.test.ts b/src/core/server/elasticsearch/elasticsearch_service.test.ts index 5a7d223fec7ad..022a03e01d37d 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.test.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.test.ts @@ -31,6 +31,7 @@ import { httpServiceMock } from '../http/http_service.mock'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; import { elasticsearchServiceMock } from './elasticsearch_service.mock'; +import { duration } from 'moment'; let elasticsearchService: ElasticsearchService; const configService = configServiceMock.create(); @@ -41,7 +42,7 @@ configService.atPath.mockReturnValue( new BehaviorSubject({ hosts: ['http://1.2.3.4'], healthCheck: { - delay: 2000, + delay: duration(2000), }, ssl: { verificationMode: 'none', @@ -125,7 +126,7 @@ describe('#setup', () => { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://8.8.8.8", ], @@ -150,7 +151,7 @@ Object { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://1.2.3.4", ], @@ -174,7 +175,7 @@ Object { new BehaviorSubject({ hosts: ['http://1.2.3.4', 'http://9.8.7.6'], healthCheck: { - delay: 2000, + delay: duration(2000), }, ssl: { verificationMode: 'none', @@ -196,7 +197,7 @@ Object { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://8.8.8.8", ], From 79202cc23c7514e9872bb542d1d37eb36690f269 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Sun, 26 Jan 2020 21:27:04 +0100 Subject: [PATCH 05/19] Wait till for compatible ES nodes before SO migrations --- .../saved_objects_service.test.ts | 37 ++++++++++++++++++- .../saved_objects/saved_objects_service.ts | 10 ++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts index 6668d57045a95..214250d6e5ed5 100644 --- a/src/core/server/saved_objects/saved_objects_service.test.ts +++ b/src/core/server/saved_objects/saved_objects_service.test.ts @@ -31,11 +31,14 @@ import { configServiceMock } from '../mocks'; import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock'; import { legacyServiceMock } from '../legacy/legacy_service.mock'; import { SavedObjectsClientFactoryProvider } from './service/lib'; +import { BehaviorSubject } from 'rxjs'; +import { NodesVersionCompatibility } from '../elasticsearch/version_check/ensure_es_version'; describe('SavedObjectsService', () => { const createSetupDeps = () => { + const elasticsearchMock = elasticsearchServiceMock.createInternalSetup(); return { - elasticsearch: elasticsearchServiceMock.createInternalSetup(), + elasticsearch: elasticsearchMock, legacyPlugins: legacyServiceMock.createDiscoverPlugins(), }; }; @@ -149,6 +152,38 @@ describe('SavedObjectsService', () => { expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(true); }); + it('waits for all es nodes to be compatible before running migrations', async done => { + expect.assertions(3); + const configService = configServiceMock.create({ atPath: { skip: false } }); + const coreContext = mockCoreContext.create({ configService }); + const soService = new SavedObjectsService(coreContext); + const setupDeps = createSetupDeps(); + // Create an new subject so that we can control when isCompatible=true + // is emitted. + setupDeps.elasticsearch.esNodesCompatibility$ = new BehaviorSubject({ + isCompatible: false, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + await soService.setup(setupDeps); + soService.start({}); + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(0); + ((setupDeps.elasticsearch.esNodesCompatibility$ as any) as BehaviorSubject< + NodesVersionCompatibility + >).next({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + setImmediate(() => { + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(false); + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1); + done(); + }); + }); + it('resolves with KibanaMigrator after waiting for migrations to complete', async () => { const configService = configServiceMock.create({ atPath: { skip: false } }); const coreContext = mockCoreContext.create({ configService }); diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index b08033a19242b..7c99e61533816 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -18,7 +18,7 @@ */ import { CoreService } from 'src/core/types'; -import { first } from 'rxjs/operators'; +import { first, filter, take } from 'rxjs/operators'; import { SavedObjectsClient, SavedObjectsSchema, @@ -283,6 +283,14 @@ export class SavedObjectsService const cliArgs = this.coreContext.env.cliArgs; const skipMigrations = cliArgs.optimize || savedObjectsConfig.skip; + this.logger.debug( + 'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...' + ); + await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe( + filter(nodes => nodes.isCompatible), + take(1) + ).toPromise(); + this.logger.debug('Starting saved objects migration'); await migrator.runMigrations(skipMigrations); this.logger.debug('Saved objects migration completed'); From a67bd797148c6ad494398938bf1b187c578a9074 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Mon, 27 Jan 2020 14:39:01 +0100 Subject: [PATCH 06/19] Fix typo --- src/legacy/core_plugins/elasticsearch/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/legacy/core_plugins/elasticsearch/index.js b/src/legacy/core_plugins/elasticsearch/index.js index 773d48f42917d..55e81e4144bb5 100644 --- a/src/legacy/core_plugins/elasticsearch/index.js +++ b/src/legacy/core_plugins/elasticsearch/index.js @@ -95,7 +95,7 @@ export default function(kibana) { const waitUntilHealthy = versionHealthCheck( this, server.logWithMetadata, - server.newPlatform.__internals.elasticsearch.esNodesCompatibilty$ + server.newPlatform.__internals.elasticsearch.esNodesCompatibility$ ); server.expose('waitUntilReady', () => waitUntilHealthy); From 9b3b9a547d6cc617504cf304fb1136a215d7743a Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Mon, 27 Jan 2020 14:50:48 +0100 Subject: [PATCH 07/19] Don't wait for ES compatibility if skipMigrations=true --- .../migrations/kibana/kibana_migrator.test.ts | 6 ----- .../migrations/kibana/kibana_migrator.ts | 15 +++-------- .../saved_objects_service.test.ts | 8 +++--- .../saved_objects/saved_objects_service.ts | 26 ++++++++++++------- 4 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index b89abc596ad18..c6a72eb53d6c4 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -59,12 +59,6 @@ describe('KibanaMigrator', () => { }); describe('runMigrations', () => { - it('resolves isMigrated if migrations were skipped', async () => { - const skipMigrations = true; - const result = await new KibanaMigrator(mockOptions()).runMigrations(skipMigrations); - expect(result).toEqual([{ status: 'skipped' }, { status: 'skipped' }]); - }); - it('only runs migrations once if called multiple times', async () => { const options = mockOptions(); const clusterStub = jest.fn(() => ({ status: 404 })); diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index c35e8dd90b5b1..747b48a540109 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -107,24 +107,15 @@ export class KibanaMigrator { * The promise resolves with an array of migration statuses, one for each * elasticsearch index which was migrated. */ - public runMigrations(skipMigrations: boolean = false): Promise> { + public runMigrations(): Promise> { if (this.migrationResult === undefined) { - this.migrationResult = this.runMigrationsInternal(skipMigrations); + this.migrationResult = this.runMigrationsInternal(); } return this.migrationResult; } - private runMigrationsInternal(skipMigrations: boolean) { - if (skipMigrations) { - this.log.warn( - 'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.' - ); - return Promise.resolve( - Object.keys(this.mappingProperties).map(() => ({ status: 'skipped' })) - ); - } - + private runMigrationsInternal() { const kibanaIndexName = this.kibanaConfig.index; const indexMap = createIndexMap({ config: this.config, diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts index 214250d6e5ed5..19798aa68928d 100644 --- a/src/core/server/saved_objects/saved_objects_service.test.ts +++ b/src/core/server/saved_objects/saved_objects_service.test.ts @@ -140,7 +140,7 @@ describe('SavedObjectsService', () => { await soService.setup(createSetupDeps()); await soService.start({}); - expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(true); + expect(migratorInstanceMock.runMigrations).not.toHaveBeenCalled(); }); it('skips KibanaMigrator migrations when migrations.skip=true', async () => { @@ -149,11 +149,11 @@ describe('SavedObjectsService', () => { const soService = new SavedObjectsService(coreContext); await soService.setup(createSetupDeps()); await soService.start({}); - expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(true); + expect(migratorInstanceMock.runMigrations).not.toHaveBeenCalled(); }); it('waits for all es nodes to be compatible before running migrations', async done => { - expect.assertions(3); + expect.assertions(2); const configService = configServiceMock.create({ atPath: { skip: false } }); const coreContext = mockCoreContext.create({ configService }); const soService = new SavedObjectsService(coreContext); @@ -178,7 +178,6 @@ describe('SavedObjectsService', () => { kibanaVersion: '8.0.0', }); setImmediate(() => { - expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(false); expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1); done(); }); @@ -193,7 +192,6 @@ describe('SavedObjectsService', () => { const startContract = await soService.start({}); expect(startContract.migrator).toBe(migratorInstanceMock); - expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith(false); expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1); }); }); diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index 7c99e61533816..3919840b033e3 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -283,17 +283,23 @@ export class SavedObjectsService const cliArgs = this.coreContext.env.cliArgs; const skipMigrations = cliArgs.optimize || savedObjectsConfig.skip; - this.logger.debug( - 'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...' - ); - await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe( - filter(nodes => nodes.isCompatible), - take(1) - ).toPromise(); + if (skipMigrations) { + this.logger.warn( + 'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.' + ); + } else { + this.logger.debug( + 'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...' + ); + await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe( + filter(nodes => nodes.isCompatible), + take(1) + ).toPromise(); - this.logger.debug('Starting saved objects migration'); - await migrator.runMigrations(skipMigrations); - this.logger.debug('Saved objects migration completed'); + this.logger.debug('Starting saved objects migration'); + await migrator.runMigrations(); + this.logger.debug('Saved objects migration completed'); + } const createRepository = (callCluster: APICaller, extraTypes: string[] = []) => { return SavedObjectsRepository.createRepository( From 7966883566bd47a236725cae988cda885661b20a Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 28 Jan 2020 14:11:11 +0100 Subject: [PATCH 08/19] Legacy Elasticsearch plugin integration test --- .../core_plugins/elasticsearch/index.d.ts | 1 + .../integration_tests/elasticsearch.test.ts | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts diff --git a/src/legacy/core_plugins/elasticsearch/index.d.ts b/src/legacy/core_plugins/elasticsearch/index.d.ts index 4cbb1c82cc1e4..df713160137a6 100644 --- a/src/legacy/core_plugins/elasticsearch/index.d.ts +++ b/src/legacy/core_plugins/elasticsearch/index.d.ts @@ -523,6 +523,7 @@ export interface CallCluster { } export interface ElasticsearchPlugin { + status: { on: (status: string, cb: () => void) => void }; getCluster(name: string): Cluster; createCluster(name: string, config: ClusterConfig): Cluster; waitUntilReady(): Promise; diff --git a/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts new file mode 100644 index 0000000000000..ec2f03b5c3466 --- /dev/null +++ b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + createTestServers, + TestElasticsearchUtils, + TestKibanaUtils, + TestUtils, + createRootWithCorePlugins, + getKbnServer, +} from '../../../../test_utils/kbn_server'; + +import { BehaviorSubject } from 'rxjs'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { NodesVersionCompatibility } from 'src/core/server/elasticsearch/version_check/ensure_es_version'; + +describe('Elasticsearch plugin', () => { + let servers: TestUtils; + let esServer: TestElasticsearchUtils; + let root: TestKibanaUtils['root']; + + let kbnServer: TestKibanaUtils['kbnServer']; + let elasticsearch: TestKibanaUtils['kbnServer']['server']['plugins']['elasticsearch']; + + const esNodesCompatibility$ = new BehaviorSubject({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + + beforeAll(async function() { + const settings = { + elasticsearch: {}, + adjustTimeout: (t: any) => { + jest.setTimeout(t); + }, + }; + servers = createTestServers(settings); + esServer = await servers.startES(); + + const elasticsearchSettings = { + hosts: esServer.hosts, + username: esServer.username, + password: esServer.password, + }; + root = createRootWithCorePlugins({ elasticsearch: elasticsearchSettings }); + + const setup = await root.setup(); + setup.elasticsearch.esNodesCompatibility$ = esNodesCompatibility$; + await root.start(); + + elasticsearch = getKbnServer(root).server.plugins.elasticsearch; + }); + + afterAll(async () => { + await esServer.stop(); + await root.shutdown(); + }, 30000); + + it("should set it's status to green when all nodes are compatible", done => { + jest.setTimeout(30000); + elasticsearch.status.on('green', () => done()); + }); + + it("should set it's status to red when some nodes aren't compatible", done => { + esNodesCompatibility$.next({ + isCompatible: false, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + elasticsearch.status.on('red', () => done()); + }); +}); From 2540de81e7a3476aa945d355690e6f6cbff734d3 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 28 Jan 2020 14:12:37 +0100 Subject: [PATCH 09/19] Fix integration tests --- .../server/http/integration_tests/core_services.test.ts | 6 +++--- .../server/legacy/integration_tests/legacy_service.test.ts | 2 +- .../http/integration_tests/default_route_provider.test.ts | 2 +- .../integration_tests/default_route_provider_config.test.ts | 1 + .../server/http/integration_tests/max_payload_size.test.js | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts index 65c4f1432721d..0d4570a34ca3c 100644 --- a/src/core/server/http/integration_tests/core_services.test.ts +++ b/src/core/server/http/integration_tests/core_services.test.ts @@ -46,7 +46,7 @@ describe('http service', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => { @@ -180,7 +180,7 @@ describe('http service', () => { describe('#basePath()', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => await root.shutdown()); @@ -209,7 +209,7 @@ describe('http service', () => { describe('elasticsearch', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => { diff --git a/src/core/server/legacy/integration_tests/legacy_service.test.ts b/src/core/server/legacy/integration_tests/legacy_service.test.ts index da2550f2ae799..e8bcf7a42d192 100644 --- a/src/core/server/legacy/integration_tests/legacy_service.test.ts +++ b/src/core/server/legacy/integration_tests/legacy_service.test.ts @@ -22,7 +22,7 @@ describe('legacy service', () => { describe('http server', () => { let root: ReturnType; beforeEach(() => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => await root.shutdown()); diff --git a/src/legacy/server/http/integration_tests/default_route_provider.test.ts b/src/legacy/server/http/integration_tests/default_route_provider.test.ts index 4898cb2b67852..d91438d904558 100644 --- a/src/legacy/server/http/integration_tests/default_route_provider.test.ts +++ b/src/legacy/server/http/integration_tests/default_route_provider.test.ts @@ -29,7 +29,7 @@ let mockDefaultRouteSetting: any = ''; describe('default route provider', () => { let root: Root; beforeAll(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); await root.setup(); await root.start(); diff --git a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts index da785a59893ab..8365941cbeb10 100644 --- a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts +++ b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts @@ -30,6 +30,7 @@ describe('default route provider', () => { server: { defaultRoute: '/app/some/default/route', }, + migrations: { skip: true }, }); await root.setup(); diff --git a/src/legacy/server/http/integration_tests/max_payload_size.test.js b/src/legacy/server/http/integration_tests/max_payload_size.test.js index 4408f0141bb21..7f22f83c78f0e 100644 --- a/src/legacy/server/http/integration_tests/max_payload_size.test.js +++ b/src/legacy/server/http/integration_tests/max_payload_size.test.js @@ -21,7 +21,7 @@ import * as kbnTestServer from '../../../../test_utils/kbn_server'; let root; beforeAll(async () => { - root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 } }); + root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 }, migrations: { skip: true } }); await root.setup(); await root.start(); From 816fd4db9a48d8ff63b9e431e701b3621ff1a3d5 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 28 Jan 2020 15:51:18 +0100 Subject: [PATCH 10/19] Fix more tests --- .../elasticsearch/integration_tests/elasticsearch.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts index ec2f03b5c3466..5806c31b78414 100644 --- a/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts +++ b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts @@ -34,8 +34,6 @@ describe('Elasticsearch plugin', () => { let servers: TestUtils; let esServer: TestElasticsearchUtils; let root: TestKibanaUtils['root']; - - let kbnServer: TestKibanaUtils['kbnServer']; let elasticsearch: TestKibanaUtils['kbnServer']['server']['plugins']['elasticsearch']; const esNodesCompatibility$ = new BehaviorSubject({ From 61e89861b9235beccc4c3442b8050cc76ce5e83c Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 28 Jan 2020 21:03:23 +0100 Subject: [PATCH 11/19] Fix spaces tests --- .../request_interceptors/on_post_auth_interceptor.test.ts | 5 ++++- .../lib/request_interceptors/on_request_interceptor.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts index c1f557f164ad6..b62211440bb7a 100644 --- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts +++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts @@ -107,7 +107,10 @@ describe('onPostAuthInterceptor', () => { availableSpaces: any[], testOptions = { simulateGetSpacesFailure: false, simulateGetSingleSpaceFailure: false } ) { - const { http } = await root.setup(); + const { http, elasticsearch } = await root.setup(); + + // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check + elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$; const loggingMock = loggingServiceMock .create() diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts index d6ff4a20052e4..5e6cf67ee8c90 100644 --- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts +++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts @@ -17,6 +17,7 @@ import { import * as kbnTestServer from '../../../../../../src/test_utils/kbn_server'; import { LegacyAPI } from '../../plugin'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; describe('onRequestInterceptor', () => { let root: ReturnType; @@ -104,7 +105,9 @@ describe('onRequestInterceptor', () => { routes: 'legacy' | 'new-platform'; } async function setup(opts: SetupOpts = { basePath: '/', routes: 'legacy' }) { - const { http } = await root.setup(); + const { http, elasticsearch } = await root.setup(); + // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check + elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$; initSpacesOnRequestInterceptor({ getLegacyAPI: () => From 134f221106d34c8927824949dcdef687ab5b9301 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Wed, 29 Jan 2020 13:14:36 +0100 Subject: [PATCH 12/19] Refactoring and PR feedback --- .../elasticsearch/elasticsearch_config.ts | 2 +- .../elasticsearch/elasticsearch_service.ts | 7 ++ .../version_check/ensure_es_version.test.ts | 4 +- .../version_check/ensure_es_version.ts | 69 +++++++++++-------- ...> es_kibana_version_compatability.test.ts} | 16 ++--- ....ts => es_kibana_version_compatability.ts} | 15 ++-- .../elasticsearch/lib/version_health_check.js | 2 +- 7 files changed, 68 insertions(+), 47 deletions(-) rename src/core/server/elasticsearch/version_check/{is_es_compatible_with_kibana.test.ts => es_kibana_version_compatability.test.ts} (72%) rename src/core/server/elasticsearch/version_check/{is_es_compatible_with_kibana.ts => es_kibana_version_compatability.ts} (76%) diff --git a/src/core/server/elasticsearch/elasticsearch_config.ts b/src/core/server/elasticsearch/elasticsearch_config.ts index c0fe3fd172b20..50866e5550d8e 100644 --- a/src/core/server/elasticsearch/elasticsearch_config.ts +++ b/src/core/server/elasticsearch/elasticsearch_config.ts @@ -106,7 +106,7 @@ const configSchema = schema.object({ ignoreVersionMismatch: schema.conditional( schema.contextRef('dev'), false, - schema.any({ + schema.boolean({ validate: rawValue => { if (rawValue === true) { return '"ignoreVersionMismatch" can only be set to true in development mode'; diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts index 44d261a03b2c9..4d897dbd8f7aa 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.ts @@ -170,6 +170,13 @@ export class ElasticsearchService implements CoreService).connect(); + // TODO: Move to Status Service https://github.com/elastic/kibana/issues/41983 + esNodesCompatibility$.subscribe(({ isCompatible, message }) => { + if (!isCompatible && message) { + this.log.error(message); + } + }); + return { legacy: { config$: clients$.pipe(map(clients => clients.config)) }, diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts index 4c87773d2fcf8..addcca3133f12 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts @@ -111,7 +111,7 @@ describe('mapNodesVersionCompatibility', () => { describe('pollEsNodesVersion', () => { const callWithInternalUser = jest.fn(); it('keeps polling when a poll request throws', done => { - expect.assertions(2); + expect.assertions(3); callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); callWithInternalUser.mockRejectedValueOnce(new Error('mock request error')); callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.1.1-Beta1')); @@ -122,7 +122,7 @@ describe('pollEsNodesVersion', () => { kibanaVersion: KIBANA_VERSION, log: mockLogger, }) - .pipe(take(2)) + .pipe(take(3)) .subscribe({ next: result => expect(result.isCompatible).toBeDefined(), complete: done, diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index 238daa1760abc..0203df124ba14 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -22,10 +22,12 @@ * that defined in Kibana's package.json. */ -import { coerce } from 'semver'; -import { interval } from 'rxjs'; -import { map, switchMap, catchError, distinctUntilChanged } from 'rxjs/operators'; -import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; +import { interval, from } from 'rxjs'; +import { map, switchMap, distinctUntilChanged, catchError, startWith } from 'rxjs/operators'; +import { + esVersionCompatibleWithKibana, + esVersionEqualsKibana, +} from './es_kibana_version_compatability'; import { Logger } from '../../logging'; import { APICaller } from '..'; @@ -37,12 +39,6 @@ export interface PollEsNodesVersionOptions { esVersionCheckInterval: number; } -export interface NodesInfo { - nodes: { - [key: string]: NodeInfo; - }; -} - interface NodeInfo { version: string; ip: string; @@ -52,6 +48,12 @@ interface NodeInfo { name: string; } +export interface NodesInfo { + nodes: { + [key: string]: NodeInfo; + }; +} + export interface NodesVersionCompatibility { isCompatible: boolean; message?: string; @@ -77,18 +79,14 @@ export function mapNodesVersionCompatibility( // Aggregate incompatible ES nodes. const incompatibleNodes = nodes.filter( - node => !isEsCompatibleWithKibana(node.version, kibanaVersion) + node => !esVersionCompatibleWithKibana(node.version, kibanaVersion) ); // Aggregate ES nodes which should prompt a Kibana upgrade. It's acceptable - // if ES and Kibana versions are not the same so long as they are not + // if ES and Kibana versions are not the same as long as they are not // incompatible, but we should warn about it. // Ignore version qualifiers https://github.com/elastic/elasticsearch/issues/36859 - const warningNodes = nodes.filter(node => { - const nodeSemVer = coerce(node.version); - const kibanaSemver = coerce(kibanaVersion); - return nodeSemVer && kibanaSemver && nodeSemVer.version !== kibanaSemver.version; - }); + const warningNodes = nodes.filter(node => !esVersionEqualsKibana(node.version, kibanaVersion)); let message; if (incompatibleNodes.length > 0) { @@ -115,6 +113,18 @@ export function mapNodesVersionCompatibility( }; } +// Returns true if two NodesVersionCompatibility entries match +function compareNodes(prev: NodesVersionCompatibility, curr: NodesVersionCompatibility) { + const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version; + return ( + curr.isCompatible === prev.isCompatible && + curr.incompatibleNodes.length === prev.incompatibleNodes.length && + curr.warningNodes.length === prev.warningNodes.length && + curr.incompatibleNodes.every((node, i) => nodesEqual(node, prev.incompatibleNodes[i])) && + curr.warningNodes.every((node, i) => nodesEqual(node, prev.warningNodes[i])) + ); +} + export const pollEsNodesVersion = ({ callWithInternalUser, log, @@ -130,22 +140,21 @@ export const pollEsNodesVersion = ({ filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], }); }), - // Log, but otherwise ignore 'nodes.info' request errors - catchError((err, caught$) => { - log.error('Unable to retrieve version information from Elasticsearch nodes.', err); - return caught$; - }), map((nodesInfo: NodesInfo) => mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch) ), - // Only emit if the IP or version numbers of the nodes changed from the - // previous result. - distinctUntilChanged((prev, curr) => { - const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version; - return ( - curr.incompatibleNodes.every((node, i) => nodesEqual(node, prev.incompatibleNodes[i])) && - curr.warningNodes.every((node, i) => nodesEqual(node, prev.warningNodes[i])) + catchError((_err, caught$) => { + // Return `isCompatible=false` when there's a 'nodes.info' request error + return caught$.pipe( + startWith({ + isCompatible: false, + message: 'Unable to retrieve version information from Elasticsearch nodes.', + incompatibleNodes: [], + warningNodes: [], + kibanaVersion, + }) ); - }) + }), + distinctUntilChanged(compareNodes) // Only emit if there are new nodes or versions ); }; diff --git a/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.test.ts b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts similarity index 72% rename from src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.test.ts rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts index 374ec240b2f49..152f25c813881 100644 --- a/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.test.ts +++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts @@ -17,39 +17,39 @@ * under the License. */ -import { isEsCompatibleWithKibana } from './is_es_compatible_with_kibana'; +import { esVersionCompatibleWithKibana } from './es_kibana_version_compatability'; describe('plugins/elasticsearch', () => { describe('lib/is_es_compatible_with_kibana', () => { describe('returns false', () => { it('when ES major is greater than Kibana major', () => { - expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).toBe(false); + expect(esVersionCompatibleWithKibana('1.0.0', '0.0.0')).toBe(false); }); it('when ES major is less than Kibana major', () => { - expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).toBe(false); + expect(esVersionCompatibleWithKibana('0.0.0', '1.0.0')).toBe(false); }); it('when majors are equal, but ES minor is less than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).toBe(false); + expect(esVersionCompatibleWithKibana('1.0.0', '1.1.0')).toBe(false); }); }); describe('returns true', () => { it('when version numbers are the same', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).toBe(true); + expect(esVersionCompatibleWithKibana('1.1.1', '1.1.1')).toBe(true); }); it('when majors are equal, and ES minor is greater than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).toBe(true); + expect(esVersionCompatibleWithKibana('1.1.0', '1.0.0')).toBe(true); }); it('when majors and minors are equal, and ES patch is greater than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).toBe(true); + expect(esVersionCompatibleWithKibana('1.1.1', '1.1.0')).toBe(true); }); it('when majors and minors are equal, but ES patch is less than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).toBe(true); + expect(esVersionCompatibleWithKibana('1.1.0', '1.1.1')).toBe(true); }); }); }); diff --git a/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.ts b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts similarity index 76% rename from src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.ts rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts index fd0bdd29ff837..28b9c0a23e672 100644 --- a/src/core/server/elasticsearch/version_check/is_es_compatible_with_kibana.ts +++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts @@ -17,15 +17,14 @@ * under the License. */ +import semver, { coerce } from 'semver'; + /** - * Let's weed out the ES versions that won't work with a given Kibana version. + * Checks for the compatibilitiy between Elasticsearch and Kibana versions * 1. Major version differences will never work together. * 2. Older versions of ES won't work with newer versions of Kibana. */ - -import semver from 'semver'; - -export function isEsCompatibleWithKibana(esVersion: string, kibanaVersion: string) { +export function esVersionCompatibleWithKibana(esVersion: string, kibanaVersion: string) { const esVersionNumbers = { major: semver.major(esVersion), minor: semver.minor(esVersion), @@ -50,3 +49,9 @@ export function isEsCompatibleWithKibana(esVersion: string, kibanaVersion: strin return true; } + +export function esVersionEqualsKibana(nodeVersion: string, kibanaVersion: string) { + const nodeSemVer = coerce(nodeVersion); + const kibanaSemver = coerce(kibanaVersion); + return nodeSemVer && kibanaSemver && nodeSemVer.version === kibanaSemver.version; +} diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js index 5eaafccd7b8d9..4ee8307f490eb 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js +++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js @@ -25,7 +25,7 @@ export const versionHealthCheck = (esPlugin, logWithMetadata, esNodesCompatibili if (!isCompatible) { esPlugin.status.red(message); } else { - if (message && message.length > 0) { + if (message) { logWithMetadata(['warning'], message, { kibanaVersion, nodes: warningNodes, From e51b5b05e3c7a2451bb4d69d0c333ddfedac4b2e Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Wed, 29 Jan 2020 15:47:04 +0100 Subject: [PATCH 13/19] Make ES compatibility check and migrations logging more visible --- src/core/server/saved_objects/saved_objects_service.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index 3919840b033e3..0c985c71c7e2f 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -288,7 +288,7 @@ export class SavedObjectsService 'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.' ); } else { - this.logger.debug( + this.logger.info( 'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...' ); await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe( @@ -296,9 +296,8 @@ export class SavedObjectsService take(1) ).toPromise(); - this.logger.debug('Starting saved objects migration'); + this.logger.info('Starting saved objects migrations'); await migrator.runMigrations(); - this.logger.debug('Saved objects migration completed'); } const createRepository = (callCluster: APICaller, extraTypes: string[] = []) => { @@ -357,14 +356,14 @@ export class SavedObjectsService savedObjectMappings: this.mappings, savedObjectMigrations: this.migrations, savedObjectValidations: this.validations, - logger: this.coreContext.logger.get('migrations'), + logger: this.logger, kibanaVersion: this.coreContext.env.packageInfo.version, config: this.setupDeps!.legacyPlugins.pluginExtendedConfig, savedObjectsConfig, kibanaConfig, callCluster: migrationsRetryCallCluster( adminClient.callAsInternalUser, - this.coreContext.logger.get('migrations'), + this.logger, migrationsRetryDelay ), }); From c5e754dd0a0cabf5cf73e7a4f2149ec390b4efdf Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Wed, 29 Jan 2020 17:17:56 +0100 Subject: [PATCH 14/19] Fix type errors --- .../server/elasticsearch/version_check/ensure_es_version.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index 0203df124ba14..ddab22c0cee9d 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -22,7 +22,7 @@ * that defined in Kibana's package.json. */ -import { interval, from } from 'rxjs'; +import { interval } from 'rxjs'; import { map, switchMap, distinctUntilChanged, catchError, startWith } from 'rxjs/operators'; import { esVersionCompatibleWithKibana, From b990cb7acd45aa6fc09e1a6b5eae1af9d9df4340 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 30 Jan 2020 14:57:26 +0100 Subject: [PATCH 15/19] Fix tests after merge with master --- src/core/server/http/integration_tests/core_services.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts index 831d403ef642d..425d8cac1893e 100644 --- a/src/core/server/http/integration_tests/core_services.test.ts +++ b/src/core/server/http/integration_tests/core_services.test.ts @@ -43,7 +43,7 @@ describe('http service', () => { describe('auth', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => { From b120ef3dd6683c7dca3ef75a40e4c85b1dbd1d42 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 30 Jan 2020 15:40:51 +0100 Subject: [PATCH 16/19] Test for isCompatible=false when ES version check throws --- .../elasticsearch/version_check/ensure_es_version.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts index addcca3133f12..6d531b383775c 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts @@ -110,7 +110,8 @@ describe('mapNodesVersionCompatibility', () => { describe('pollEsNodesVersion', () => { const callWithInternalUser = jest.fn(); - it('keeps polling when a poll request throws', done => { + const expectedCompatibilityResults = [false, false, true]; + it('returns iscCompatible=false and keeps polling when a poll request throws', done => { expect.assertions(3); callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); callWithInternalUser.mockRejectedValueOnce(new Error('mock request error')); @@ -124,7 +125,9 @@ describe('pollEsNodesVersion', () => { }) .pipe(take(3)) .subscribe({ - next: result => expect(result.isCompatible).toBeDefined(), + next: result => { + expect(result.isCompatible).toBe(expectedCompatibilityResults.shift()); + }, complete: done, error: done, }); From 164324def882bb9574d8256d819664c49793794a Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 30 Jan 2020 15:41:32 +0100 Subject: [PATCH 17/19] add comment: Incompatibility message takes precedence --- .../server/elasticsearch/version_check/ensure_es_version.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index ddab22c0cee9d..a4467bc5802a4 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -88,6 +88,8 @@ export function mapNodesVersionCompatibility( // Ignore version qualifiers https://github.com/elastic/elasticsearch/issues/36859 const warningNodes = nodes.filter(node => !esVersionEqualsKibana(node.version, kibanaVersion)); + // Note: If incompatible and warning nodes are present `message` only contains + // an incompatibility notice. let message; if (incompatibleNodes.length > 0) { const incompatibleNodeNames = incompatibleNodes.map(node => node.name).join(', '); From 91e0256e2f34f5a0670975c0bf691ad0be4f958f Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 30 Jan 2020 23:07:08 +0100 Subject: [PATCH 18/19] Start pollEsNodesVersion immediately --- .../server/elasticsearch/version_check/ensure_es_version.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index a4467bc5802a4..362b59cc1027c 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -22,7 +22,7 @@ * that defined in Kibana's package.json. */ -import { interval } from 'rxjs'; +import { timer } from 'rxjs'; import { map, switchMap, distinctUntilChanged, catchError, startWith } from 'rxjs/operators'; import { esVersionCompatibleWithKibana, @@ -136,7 +136,7 @@ export const pollEsNodesVersion = ({ }: PollEsNodesVersionOptions) => { log.debug('Checking Elasticsearch version'); - return interval(healthCheckInterval).pipe( + return timer(0, healthCheckInterval).pipe( switchMap(() => { return callWithInternalUser('nodes.info', { filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], From f05eab1e1367fe04fc9c87ea2d500807436b3477 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 30 Jan 2020 23:41:02 +0100 Subject: [PATCH 19/19] Refactor pollEsNodesVersion --- .../version_check/ensure_es_version.test.ts | 85 ++++++++++++++++++- .../version_check/ensure_es_version.ts | 42 ++++----- 2 files changed, 105 insertions(+), 22 deletions(-) diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts index 6d531b383775c..4989c4a31295c 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts @@ -18,7 +18,9 @@ */ import { mapNodesVersionCompatibility, pollEsNodesVersion, NodesInfo } from './ensure_es_version'; import { loggingServiceMock } from '../../logging/logging_service.mock'; -import { take } from 'rxjs/operators'; +import { take, delay } from 'rxjs/operators'; +import { TestScheduler } from 'rxjs/testing'; +import { of } from 'rxjs'; const mockLoggerFactory = loggingServiceMock.create(); const mockLogger = mockLoggerFactory.get('mock logger'); @@ -110,9 +112,19 @@ describe('mapNodesVersionCompatibility', () => { describe('pollEsNodesVersion', () => { const callWithInternalUser = jest.fn(); - const expectedCompatibilityResults = [false, false, true]; + const getTestScheduler = () => + new TestScheduler((actual, expected) => { + expect(actual).toEqual(expected); + }); + + beforeEach(() => { + callWithInternalUser.mockClear(); + }); + it('returns iscCompatible=false and keeps polling when a poll request throws', done => { expect.assertions(3); + const expectedCompatibilityResults = [false, false, true]; + jest.clearAllMocks(); callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); callWithInternalUser.mockRejectedValueOnce(new Error('mock request error')); callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.1.1-Beta1')); @@ -177,4 +189,73 @@ describe('pollEsNodesVersion', () => { error: done, }); }); + + it('starts polling immediately and then every esVersionCheckInterval', () => { + expect.assertions(1); + callWithInternalUser.mockReturnValueOnce([createNodes('5.1.0', '5.2.0', '5.0.0')]); + callWithInternalUser.mockReturnValueOnce([createNodes('5.1.1', '5.2.0', '5.0.0')]); + + getTestScheduler().run(({ expectObservable }) => { + const expected = 'a 99ms (b|)'; + + const esNodesCompatibility$ = pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 100, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }).pipe(take(2)); + + expectObservable(esNodesCompatibility$).toBe(expected, { + a: mapNodesVersionCompatibility( + createNodes('5.1.0', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + b: mapNodesVersionCompatibility( + createNodes('5.1.1', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + }); + }); + }); + + it('waits for es version check requests to complete before scheduling the next one', () => { + expect.assertions(2); + + getTestScheduler().run(({ expectObservable }) => { + const expected = '100ms a 99ms (b|)'; + + callWithInternalUser.mockReturnValueOnce( + of(createNodes('5.1.0', '5.2.0', '5.0.0')).pipe(delay(100)) + ); + callWithInternalUser.mockReturnValueOnce( + of(createNodes('5.1.1', '5.2.0', '5.0.0')).pipe(delay(100)) + ); + + const esNodesCompatibility$ = pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 10, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }).pipe(take(2)); + + expectObservable(esNodesCompatibility$).toBe(expected, { + a: mapNodesVersionCompatibility( + createNodes('5.1.0', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + b: mapNodesVersionCompatibility( + createNodes('5.1.1', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + }); + }); + + expect(callWithInternalUser).toHaveBeenCalledTimes(2); + }); }); diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts index 362b59cc1027c..3e760ec0efabd 100644 --- a/src/core/server/elasticsearch/version_check/ensure_es_version.ts +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -22,8 +22,8 @@ * that defined in Kibana's package.json. */ -import { timer } from 'rxjs'; -import { map, switchMap, distinctUntilChanged, catchError, startWith } from 'rxjs/operators'; +import { timer, of, from, Observable } from 'rxjs'; +import { map, distinctUntilChanged, catchError, exhaustMap } from 'rxjs/operators'; import { esVersionCompatibleWithKibana, esVersionEqualsKibana, @@ -72,6 +72,15 @@ export function mapNodesVersionCompatibility( kibanaVersion: string, ignoreVersionMismatch: boolean ): NodesVersionCompatibility { + if (Object.keys(nodesInfo.nodes).length === 0) { + return { + isCompatible: false, + message: 'Unable to retrieve version information from Elasticsearch nodes.', + incompatibleNodes: [], + warningNodes: [], + kibanaVersion, + }; + } const nodes = Object.keys(nodesInfo.nodes) .sort() // Sorting ensures a stable node ordering for comparison .map(key => nodesInfo.nodes[key]) @@ -133,30 +142,23 @@ export const pollEsNodesVersion = ({ kibanaVersion, ignoreVersionMismatch, esVersionCheckInterval: healthCheckInterval, -}: PollEsNodesVersionOptions) => { +}: PollEsNodesVersionOptions): Observable => { log.debug('Checking Elasticsearch version'); - return timer(0, healthCheckInterval).pipe( - switchMap(() => { - return callWithInternalUser('nodes.info', { - filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], - }); + exhaustMap(() => { + return from( + callWithInternalUser('nodes.info', { + filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], + }) + ).pipe( + catchError(_err => { + return of({ nodes: {} }); + }) + ); }), map((nodesInfo: NodesInfo) => mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch) ), - catchError((_err, caught$) => { - // Return `isCompatible=false` when there's a 'nodes.info' request error - return caught$.pipe( - startWith({ - isCompatible: false, - message: 'Unable to retrieve version information from Elasticsearch nodes.', - incompatibleNodes: [], - warningNodes: [], - kibanaVersion, - }) - ); - }), distinctUntilChanged(compareNodes) // Only emit if there are new nodes or versions ); };