Merge pull request #5183 from epixa/4342-preflight-field-stats

Create indexList via field stats for wildcard patterns

src/plugins/elasticsearch/index.js

@@ -36,6 +36,7 @@ module.exports = function (kibana) {
       createProxy(server, 'GET', '/{paths*}');
       createProxy(server, 'POST', '/_mget');
       createProxy(server, 'POST', '/{index}/_search');
+      createProxy(server, 'POST', '/{index}/_field_stats');
       createProxy(server, 'POST', '/_msearch');
 
       function noBulkCheck(request, reply) {

src/ui/public/index_patterns/__tests__/_index_pattern.js

@@ -15,9 +15,13 @@ describe('index pattern', function () {
   var docSourceResponse;
   var indexPatternId = 'test-pattern';
   var indexPattern;
+  var calculateIndices;
+  var $rootScope;
+  var intervals;
 
   beforeEach(ngMock.module('kibana'));
   beforeEach(ngMock.inject(function (Private, $injector, _config_) {
+    $rootScope = $injector.get('$rootScope');
     config = _config_;
     mockLogstashFields = Private(require('fixtures/logstash_fields'));
     docSourceResponse = Private(require('fixtures/stubbed_doc_source_response'));
@@ -38,6 +42,16 @@ describe('index pattern', function () {
       return Promise.resolve(true);
     });
 
+    // stub calculateIndices
+    calculateIndices = sinon.spy(function () {
+      return $injector.get('Promise').resolve(['foo', 'bar']);
+    });
+    Private.stub(require('ui/index_patterns/_calculate_indices'), calculateIndices);
+
+    // spy on intervals
+    intervals = Private(require('ui/index_patterns/_intervals'));
+    sinon.stub(intervals, 'toIndexList').returns(['foo', 'bar']);
+
     IndexPattern = Private(require('ui/index_patterns/_index_pattern'));
   }));
 
@@ -273,4 +287,105 @@ describe('index pattern', function () {
       });
     });
   });
+
+  describe('#toIndexList', function () {
+    context('when index pattern is an interval', function () {
+      var interval;
+      beforeEach(function () {
+        interval = 'result:getInterval';
+        sinon.stub(indexPattern, 'getInterval').returns(interval);
+      });
+
+      it('invokes interval toIndexList with given start/stop times', function () {
+        indexPattern.toIndexList(1, 2);
+        $rootScope.$apply();
+
+        var id = indexPattern.id;
+        expect(intervals.toIndexList.calledWith(id, interval, 1, 2)).to.be(true);
+      });
+      it('is fulfilled by the result of interval toIndexList', function () {
+        var indexList;
+        indexPattern.toIndexList().then(function (val) {
+          indexList = val;
+        });
+        $rootScope.$apply();
+
+        expect(indexList[0]).to.equal('foo');
+        expect(indexList[1]).to.equal('bar');
+      });
+    });
+
+    context('when index pattern is a time-base wildcard', function () {
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+        sinon.stub(indexPattern, 'hasTimeField').returns(true);
+        sinon.stub(indexPattern, 'isWildcard').returns(true);
+      });
+
+      it('invokes calculateIndices with given start/stop times', function () {
+        indexPattern.toIndexList(1, 2);
+        $rootScope.$apply();
+
+        var id = indexPattern.id;
+        var field = indexPattern.timeFieldName;
+        expect(calculateIndices.calledWith(id, field, 1, 2)).to.be(true);
+      });
+      it('is fulfilled by the result of calculateIndices', function () {
+        var indexList;
+        indexPattern.toIndexList().then(function (val) {
+          indexList = val;
+        });
+        $rootScope.$apply();
+
+        expect(indexList[0]).to.equal('foo');
+        expect(indexList[1]).to.equal('bar');
+      });
+    });
+
+    context('when index pattern is neither an interval nor a time-based wildcard', function () {
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+      });
+
+      it('is fulfilled by id', function () {
+        var indexList;
+        indexPattern.toIndexList().then(function (val) {
+          indexList = val;
+        });
+        $rootScope.$apply();
+
+        expect(indexList).to.equal(indexPattern.id);
+      });
+    });
+  });
+
+  describe('#hasTimeField()', function () {
+    beforeEach(function () {
+      // for the sake of these tests, it doesn't much matter what type of field
+      // this is so long as it exists
+      indexPattern.timeFieldName = 'bytes';
+    });
+    it('returns false if no time field', function () {
+      delete indexPattern.timeFieldName;
+      expect(indexPattern.hasTimeField()).to.be(false);
+    });
+    it('returns false if time field does not actually exist in fields', function () {
+      indexPattern.timeFieldName = 'does not exist';
+      expect(indexPattern.hasTimeField()).to.be(false);
+    });
+    it('returns true if valid time field is configured', function () {
+      expect(indexPattern.hasTimeField()).to.be(true);
+    });
+  });
+
+  describe('#isWildcard()', function () {
+    it('returns true if id has an *', function () {
+      indexPattern.id = 'foo*';
+      expect(indexPattern.isWildcard()).to.be(true);
+    });
+    it('returns false if id has no *', function () {
+      indexPattern.id = 'foo';
+      expect(indexPattern.isWildcard()).to.be(false);
+    });
+  });
 });

src/ui/public/index_patterns/__tests__/calculate_indices.js

@@ -0,0 +1,98 @@
+describe('ui/index_patterns/_calculate_indices', () => {
+  const _ = require('lodash');
+  const sinon = require('auto-release-sinon');
+  const expect = require('expect.js');
+  const ngMock = require('ngMock');
+
+  let Promise;
+  let $rootScope;
+  let calculateIndices;
+  let error;
+  let response;
+  let transportRequest;
+  let config;
+  let constraints;
+
+  beforeEach(ngMock.module('kibana', ($provide) => {
+    error = undefined;
+    response = { indices: { 'mock-*': 'irrelevant, is ignored' } };
+    transportRequest = sinon.spy((options, fn) => fn(error, response));
+    $provide.value('es', _.set({}, 'transport.request', transportRequest));
+  }));
+
+  beforeEach(ngMock.inject((Private, $injector) => {
+    $rootScope = $injector.get('$rootScope');
+    Promise = $injector.get('Promise');
+    calculateIndices = Private(require('ui/index_patterns/_calculate_indices'));
+  }));
+
+  describe('transport configuration', () => {
+    it('is POST', () => {
+      run();
+      expect(config.method).to.equal('POST');
+    });
+    it('uses pattern path for _field_stats', () => {
+      run();
+      expect(config.path).to.equal('/wat-*-no/_field_stats');
+    });
+    it('has level indices', () => {
+      run();
+      expect(config.query.level).to.equal('indices');
+    });
+    it('includes time field', () => {
+      run();
+      expect(_.includes(config.body.fields, '@something')).to.be(true);
+    });
+    it('no constraints by default', () => {
+      run();
+      expect(_.size(constraints['@something'])).to.equal(0);
+    });
+
+    context('when given start', () => {
+      beforeEach(() => run({ start: '1234567890' }));
+      it('includes min_value', () => {
+        expect(constraints['@something']).to.have.property('min_value');
+      });
+      it('min_value is gte', () => {
+        expect(constraints['@something'].min_value).to.have.property('gte');
+      });
+    });
+
+    context('when given stop', () => {
+      beforeEach(() => run({ stop: '1234567890' }));
+      it('includes max_value', () => {
+        expect(constraints['@something']).to.have.property('max_value');
+      });
+      it('max_value is lt', () => {
+        expect(constraints['@something'].max_value).to.have.property('lt');
+      });
+    });
+  });
+
+  describe('returned promise', () => {
+    it('is rejected by transport errors', () => {
+      error = 'something';
+
+      let reason;
+      calculateIndices('one', 'two').then(null, val => reason = val);
+      $rootScope.$apply();
+
+      expect(reason).to.equal(error);
+    });
+    it('is fulfilled by array of indices in successful response', () => {
+
+      let indices;
+      calculateIndices('one', 'two').then(val => indices = val);
+      $rootScope.$apply();
+
+      expect(_.first(indices)).to.equal('mock-*');
+    });
+  });
+
+  function run({ start = undefined, stop = undefined } = {}) {
+    calculateIndices('wat-*-no', '@something', start, stop);
+    $rootScope.$apply();
+    config = _.first(transportRequest.firstCall.args);
+    constraints = config.body.index_constraints;
+  }
+});

src/ui/public/index_patterns/_calculate_indices.js

@@ -0,0 +1,54 @@
+define(function (require) {
+  const _ = require('lodash');
+  const moment = require('moment');
+
+  return function CalculateIndicesFactory(Promise, es) {
+
+    // Uses the field stats api to determine the names of indices that need to
+    // be queried against that match the given pattern and fall within the
+    // given time range
+    function calculateIndices(...args) {
+      const options = compileOptions(...args);
+      return sendRequest(options);
+    };
+
+    // creates the configuration hash that must be passed to the elasticsearch
+    // client
+    function compileOptions(pattern, timeFieldName, start, stop) {
+      const constraints = {};
+      if (start) {
+        constraints.min_value = { gte: moment(start).valueOf() };
+      }
+      if (stop) {
+        constraints.max_value = { lt: moment(stop).valueOf() };
+      }
+
+      return {
+        method: 'POST',
+        path: `/${pattern}/_field_stats`,
+        query: {
+          level: 'indices'
+        },
+        body: {
+          fields: [ timeFieldName ],
+          index_constraints: {
+            [timeFieldName]: constraints
+          }
+        }
+      };
+    }
+
+    // executes a request to elasticsearch with the given configuration hash
+    function sendRequest(options) {
+      return new Promise(function (resolve, reject) {
+        es.transport.request(options, function (err, response) {
+          if (err) return reject(err);
+          const indices = _.map(response.indices, (info, index) => index);
+          resolve(indices);
+        });
+      });
+    }
+
+    return calculateIndices;
+  };
+});

src/ui/public/index_patterns/_index_pattern.js

@@ -15,6 +15,7 @@ define(function (require) {
 
     var flattenHit = Private(require('ui/index_patterns/_flatten_hit'));
     var formatHit = require('ui/index_patterns/_format_hit');
+    var calculateIndices = Private(require('ui/index_patterns/_calculate_indices'));
 
     var type = 'index-pattern';
 
@@ -176,17 +177,30 @@ define(function (require) {
       };
 
       self.toIndexList = function (start, stop) {
-        var self = this;
         return new Promise(function (resolve) {
+          var indexList;
           var interval = self.getInterval();
+
           if (interval) {
-            resolve(intervals.toIndexList(self.id, interval, start, stop));
+            indexList = intervals.toIndexList(self.id, interval, start, stop);
+          } else if (self.isWildcard() && self.hasTimeField()) {
+            indexList = calculateIndices(self.id, self.timeFieldName, start, stop);
           } else {
-            resolve(self.id);
+            indexList = self.id;
           }
+
+          resolve(indexList);
         });
       };
 
+      self.hasTimeField = function () {
+        return !!(this.timeFieldName && this.fields.byName[this.timeFieldName]);
+      };
+
+      self.isWildcard = function () {
+        return _.includes(this.id, '*');
+      };
+
       self.prepBody = function () {
         var body = {};