From 41f0b2320b4858f1bc8edfdf508cd56ce2b9dab4 Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Fri, 23 Oct 2015 11:53:10 -0400 Subject: [PATCH 1/4] Create indexList via field stats for wildcard patterns Elasticsearch 2.0 introduces the concept of index constraints in the field stats api, which allows us to dynamically determine which specific indices need to be queried that match a given wildcard index pattern and fall within a specific time range. When we encounter a wildcard index pattern, we now send out a pre-flight request to the field stats api using the currently selected time range as index constraints to dynamically build the corresponding index list. --- src/plugins/elasticsearch/index.js | 1 + .../index_patterns/_calculate_indices.js | 56 +++++++++++++++++++ .../public/index_patterns/_index_pattern.js | 20 ++++++- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 src/ui/public/index_patterns/_calculate_indices.js diff --git a/src/plugins/elasticsearch/index.js b/src/plugins/elasticsearch/index.js index e193e30cec452..e33ec17d814de 100644 --- a/src/plugins/elasticsearch/index.js +++ b/src/plugins/elasticsearch/index.js @@ -36,6 +36,7 @@ module.exports = function (kibana) { createProxy(server, 'GET', '/{paths*}'); createProxy(server, 'POST', '/_mget'); createProxy(server, 'POST', '/{index}/_search'); + createProxy(server, 'POST', '/{index}/_field_stats'); createProxy(server, 'POST', '/_msearch'); function noBulkCheck(request, reply) { diff --git a/src/ui/public/index_patterns/_calculate_indices.js b/src/ui/public/index_patterns/_calculate_indices.js new file mode 100644 index 0000000000000..019ab208a8d65 --- /dev/null +++ b/src/ui/public/index_patterns/_calculate_indices.js @@ -0,0 +1,56 @@ +define(function (require) { + const _ = require('lodash'); + const moment = require('moment'); + + return function CalculateIndicesFactory(Promise, es) { + + // Uses the field stats api to determine the names of indices that need to + // be queried against that match the given pattern and fall within the + // given time range + function calculateIndices(...args) { + const options = compileOptions(...args); + return sendRequest(options); + }; + + // creates the configuration hash that must be passed to the elasticsearch + // client + function compileOptions(pattern, timeFieldName, start, stop) { + const constraints = {}; + if (start) { + constraints.min_value = { gte: moment(start).valueOf() }; + } + if (stop) { + constraints.max_value = { lt: moment(stop).valueOf() }; + } + + return { + method: 'POST', + path: `/${pattern}/_field_stats`, + query: { + level: 'indices' + }, + body: { + fields: [ timeFieldName ], + index_constraints: { + [timeFieldName]: constraints + } + } + }; + } + + // executes a request to elasticsearch with the given configuration hash + function sendRequest(options) { + return new Promise(function (resolve, reject) { + es.transport.request(options, function (err, response) { + if (err) return reject(err); + var indices = _.map(response.indices, function (info, index) { + return index; + }); + resolve(indices); + }); + }); + } + + return calculateIndices; + }; +}); diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index ba9ba8fcee76f..4b3b2a6d3c734 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -15,6 +15,7 @@ define(function (require) { var flattenHit = Private(require('ui/index_patterns/_flatten_hit')); var formatHit = require('ui/index_patterns/_format_hit'); + var calculateIndices = Private(require('ui/index_patterns/_calculate_indices')); var type = 'index-pattern'; @@ -176,17 +177,30 @@ define(function (require) { }; self.toIndexList = function (start, stop) { - var self = this; return new Promise(function (resolve) { + var indexList; var interval = self.getInterval(); + if (interval) { - resolve(intervals.toIndexList(self.id, interval, start, stop)); + indexList = intervals.toIndexList(self.id, interval, start, stop); + } else if (self.isWildcard() && self.hasTimeField()) { + indexList = calculateIndices(self.id, self.timeFieldName, start, stop); } else { - resolve(self.id); + indexList = self.id; } + + resolve(indexList); }); }; + self.hasTimeField = function () { + return !!(this.timeFieldName && this.fields.byName[this.timeFieldName]); + }; + + self.isWildcard = function () { + return _.includes(this.id, '*'); + }; + self.prepBody = function () { var body = {}; From a3950b0896c2f5c19d8fcab22c94d8424429839d Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Mon, 26 Oct 2015 16:00:34 -0400 Subject: [PATCH 2/4] Test calculate_indices --- .../__tests__/calculate_indices.js | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/ui/public/index_patterns/__tests__/calculate_indices.js diff --git a/src/ui/public/index_patterns/__tests__/calculate_indices.js b/src/ui/public/index_patterns/__tests__/calculate_indices.js new file mode 100644 index 0000000000000..304dc92ff68f4 --- /dev/null +++ b/src/ui/public/index_patterns/__tests__/calculate_indices.js @@ -0,0 +1,98 @@ +describe('ui/index_patterns/_calculate_indices', () => { + const _ = require('lodash'); + const sinon = require('auto-release-sinon'); + const expect = require('expect.js'); + const ngMock = require('ngMock'); + + let Promise; + let $rootScope; + let calculateIndices; + let error; + let response; + let transportRequest; + let config; + let constraints; + + beforeEach(ngMock.module('kibana', ($provide) => { + error = undefined; + response = { indices: { 'mock-*': 'irrelevant, is ignored' } }; + transportRequest = sinon.spy((options, fn) => fn(error, response)); + $provide.value('es', _.set({}, 'transport.request', transportRequest)); + })); + + beforeEach(ngMock.inject((Private, $injector) => { + $rootScope = $injector.get('$rootScope'); + Promise = $injector.get('Promise'); + calculateIndices = Private(require('ui/index_patterns/_calculate_indices')); + })); + + describe('transport configuration', () => { + it('is POST', () => { + run(); + expect(config.method).to.equal('POST'); + }); + it('uses pattern path for _field_stats', () => { + run(); + expect(config.path).to.equal('/wat-*-no/_field_stats'); + }); + it('has level indices', () => { + run(); + expect(config.query.level).to.equal('indices'); + }); + it('includes time field', () => { + run(); + expect(_.includes(config.body.fields, '@something')).to.be(true); + }); + it('no constraints by default', () => { + run(); + expect(_.size(constraints['@something'])).to.equal(0); + }); + + context('when given start', () => { + beforeEach(() => run({ start: '1234567890' })); + it('includes min_value', () => { + expect(constraints['@something']).to.have.property('min_value'); + }); + it('min_value is gte', () => { + expect(constraints['@something'].min_value).to.have.property('gte'); + }); + }); + + context('when given stop', () => { + beforeEach(() => run({ stop: '1234567890' })); + it('includes max_value', () => { + expect(constraints['@something']).to.have.property('max_value'); + }); + it('max_value is lt', () => { + expect(constraints['@something'].max_value).to.have.property('lt'); + }); + }); + }); + + describe('returned promise', () => { + it('is rejected by transport errors', () => { + error = 'something'; + + let reason; + calculateIndices('one', 'two').then(null, val => reason = val); + $rootScope.$apply(); + + expect(reason).to.equal(error); + }); + it('is fulfilled by array of indices in successful response', () => { + + let indices; + calculateIndices('one', 'two').then(val => indices = val); + $rootScope.$apply(); + + expect(_.first(indices)).to.equal('mock-*'); + }); + }); + + function run({ start = undefined, stop = undefined } = {}) { + calculateIndices('wat-*-no', '@something', start, stop); + $rootScope.$apply(); + config = _.first(transportRequest.firstCall.args); + constraints = config.body.index_constraints; + } +}); From ccf8c195f91d5f9d54b93994b5de4e2453b8f2ba Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Mon, 26 Oct 2015 16:03:27 -0400 Subject: [PATCH 3/4] Consistent use of es6 for calculate_indices This doesn't change the behavior of calculate_indices at all, but it is more consistent with the rest of the source code in that file. --- src/ui/public/index_patterns/_calculate_indices.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ui/public/index_patterns/_calculate_indices.js b/src/ui/public/index_patterns/_calculate_indices.js index 019ab208a8d65..bfd6dce181cc1 100644 --- a/src/ui/public/index_patterns/_calculate_indices.js +++ b/src/ui/public/index_patterns/_calculate_indices.js @@ -43,9 +43,7 @@ define(function (require) { return new Promise(function (resolve, reject) { es.transport.request(options, function (err, response) { if (err) return reject(err); - var indices = _.map(response.indices, function (info, index) { - return index; - }); + const indices = _.map(response.indices, (info, index) => index); resolve(indices); }); }); From c46abc7f7f108c6dc0c9d272364ec29b45771520 Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Mon, 26 Oct 2015 17:08:53 -0400 Subject: [PATCH 4/4] Tests for index pattern toIndexList behaviors --- .../__tests__/_index_pattern.js | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/src/ui/public/index_patterns/__tests__/_index_pattern.js b/src/ui/public/index_patterns/__tests__/_index_pattern.js index 7ab6101c009c6..f892ad98954c0 100644 --- a/src/ui/public/index_patterns/__tests__/_index_pattern.js +++ b/src/ui/public/index_patterns/__tests__/_index_pattern.js @@ -15,9 +15,13 @@ describe('index pattern', function () { var docSourceResponse; var indexPatternId = 'test-pattern'; var indexPattern; + var calculateIndices; + var $rootScope; + var intervals; beforeEach(ngMock.module('kibana')); beforeEach(ngMock.inject(function (Private, $injector, _config_) { + $rootScope = $injector.get('$rootScope'); config = _config_; mockLogstashFields = Private(require('fixtures/logstash_fields')); docSourceResponse = Private(require('fixtures/stubbed_doc_source_response')); @@ -38,6 +42,16 @@ describe('index pattern', function () { return Promise.resolve(true); }); + // stub calculateIndices + calculateIndices = sinon.spy(function () { + return $injector.get('Promise').resolve(['foo', 'bar']); + }); + Private.stub(require('ui/index_patterns/_calculate_indices'), calculateIndices); + + // spy on intervals + intervals = Private(require('ui/index_patterns/_intervals')); + sinon.stub(intervals, 'toIndexList').returns(['foo', 'bar']); + IndexPattern = Private(require('ui/index_patterns/_index_pattern')); })); @@ -273,4 +287,105 @@ describe('index pattern', function () { }); }); }); + + describe('#toIndexList', function () { + context('when index pattern is an interval', function () { + var interval; + beforeEach(function () { + interval = 'result:getInterval'; + sinon.stub(indexPattern, 'getInterval').returns(interval); + }); + + it('invokes interval toIndexList with given start/stop times', function () { + indexPattern.toIndexList(1, 2); + $rootScope.$apply(); + + var id = indexPattern.id; + expect(intervals.toIndexList.calledWith(id, interval, 1, 2)).to.be(true); + }); + it('is fulfilled by the result of interval toIndexList', function () { + var indexList; + indexPattern.toIndexList().then(function (val) { + indexList = val; + }); + $rootScope.$apply(); + + expect(indexList[0]).to.equal('foo'); + expect(indexList[1]).to.equal('bar'); + }); + }); + + context('when index pattern is a time-base wildcard', function () { + beforeEach(function () { + sinon.stub(indexPattern, 'getInterval').returns(false); + sinon.stub(indexPattern, 'hasTimeField').returns(true); + sinon.stub(indexPattern, 'isWildcard').returns(true); + }); + + it('invokes calculateIndices with given start/stop times', function () { + indexPattern.toIndexList(1, 2); + $rootScope.$apply(); + + var id = indexPattern.id; + var field = indexPattern.timeFieldName; + expect(calculateIndices.calledWith(id, field, 1, 2)).to.be(true); + }); + it('is fulfilled by the result of calculateIndices', function () { + var indexList; + indexPattern.toIndexList().then(function (val) { + indexList = val; + }); + $rootScope.$apply(); + + expect(indexList[0]).to.equal('foo'); + expect(indexList[1]).to.equal('bar'); + }); + }); + + context('when index pattern is neither an interval nor a time-based wildcard', function () { + beforeEach(function () { + sinon.stub(indexPattern, 'getInterval').returns(false); + }); + + it('is fulfilled by id', function () { + var indexList; + indexPattern.toIndexList().then(function (val) { + indexList = val; + }); + $rootScope.$apply(); + + expect(indexList).to.equal(indexPattern.id); + }); + }); + }); + + describe('#hasTimeField()', function () { + beforeEach(function () { + // for the sake of these tests, it doesn't much matter what type of field + // this is so long as it exists + indexPattern.timeFieldName = 'bytes'; + }); + it('returns false if no time field', function () { + delete indexPattern.timeFieldName; + expect(indexPattern.hasTimeField()).to.be(false); + }); + it('returns false if time field does not actually exist in fields', function () { + indexPattern.timeFieldName = 'does not exist'; + expect(indexPattern.hasTimeField()).to.be(false); + }); + it('returns true if valid time field is configured', function () { + expect(indexPattern.hasTimeField()).to.be(true); + }); + }); + + describe('#isWildcard()', function () { + it('returns true if id has an *', function () { + indexPattern.id = 'foo*'; + expect(indexPattern.isWildcard()).to.be(true); + }); + it('returns false if id has no *', function () { + indexPattern.id = 'foo'; + expect(indexPattern.isWildcard()).to.be(false); + }); + }); });