[indexPatterns/field] remove indexed/analyzed/doc_values awareness (#…

…11969) * [indexPatterns/field] replace mapping awareness with readFromDocValues * move link to more details to proper location * Address kjbekkelund's feedback * let the prs speak for themselves (cherry picked from commit 84ea502)
elastic · May 25, 2017 · 1307ea3 · 1307ea3
1 parent a3d237a
commit 1307ea3
Show file tree

Hide file tree

Showing 16 changed files with 111 additions and 113 deletions.
diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js b/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js
@@ -34,23 +34,6 @@ app.directive('discoverField', function ($compile) {
       const getWarnings = function (field) {
         let warnings = [];
 
-        if (!field.scripted) {
-          if (!field.doc_values && field.type !== 'boolean' && !(field.analyzed && field.type === 'string')) {
-            warnings.push('Doc values are not enabled on this field. This may lead to excess heap consumption when visualizing.');
-          }
-
-          if (field.analyzed && field.type === 'string') {
-            warnings.push('This is an analyzed string field.' +
-              ' Analyzed strings are highly unique and can use a lot of memory to visualize.' +
-              ' Values such as foo-bar will be broken into foo and bar.');
-          }
-
-          if (!field.indexed && !field.searchable) {
-            warnings.push('This field is not indexed and might not be usable in visualizations.');
-          }
-        }
-
-
         if (field.scripted) {
           warnings.push('Scripted fields can take a long time to execute.');
         }

diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html
@@ -62,21 +62,21 @@ <h5>Available Fields
     <form role="form" class="discover-field-details">
       <div class="form-group">
         <label>
-          Analyzed
+          Aggregatable
         </label>
         <select
           ng-options="opt.value as opt.label for opt in filter.boolOpts"
-          ng-model="filter.vals.analyzed"
+          ng-model="filter.vals.aggregatable"
           class="form-control">
         </select>
       </div>
       <div class="form-group">
         <label>
-          Indexed
+          Searchable
         </label>
         <select
           ng-options="opt.value as opt.label for opt in filter.boolOpts"
-          ng-model="filter.vals.indexed"
+          ng-model="filter.vals.searchable"
           class="form-control">
         </select>
       </div>

diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js
@@ -45,13 +45,14 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
       const filter = $scope.filter = {
         props: [
           'type',
-          'indexed',
-          'analyzed',
+          'aggregatable',
+          'searchable',
           'missing',
           'name'
         ],
         defaults: {
-          missing: true
+          missing: true,
+          type: 'any'
         },
         boolOpts: [
           { label: 'any', value: undefined },
@@ -69,16 +70,16 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
           return field.display;
         },
         isFieldFiltered: function (field) {
-          const matchFilter = (filter.vals.type == null || field.type === filter.vals.type);
-          const isAnalyzed = (filter.vals.analyzed == null || field.analyzed === filter.vals.analyzed);
-          const isIndexed = (filter.vals.indexed == null || field.indexed === filter.vals.indexed);
+          const matchFilter = (filter.vals.type === 'any' || field.type === filter.vals.type);
+          const isAggregatable = (filter.vals.aggregatable == null || field.aggregatable === filter.vals.aggregatable);
+          const isSearchable = (filter.vals.searchable == null || field.searchable === filter.vals.searchable);
           const scriptedOrMissing = (!filter.vals.missing || field.scripted || field.rowCount > 0);
           const matchName = (!filter.vals.name || field.name.indexOf(filter.vals.name) !== -1);
 
           return !field.display
             && matchFilter
-            && isAnalyzed
-            && isIndexed
+            && isAggregatable
+            && isSearchable
             && scriptedOrMissing
             && matchName
           ;
@@ -146,7 +147,7 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
         .commit();
 
         // include undefined so the user can clear the filter
-        $scope.fieldTypes = _.union([undefined], _.pluck(fields, 'type'));
+        $scope.fieldTypes = _.union(['any'], _.pluck(fields, 'type'));
       });
 
       $scope.increaseFieldCounter = function (fieldName) {

diff --git a/...nagement/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js b/...nagement/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js
@@ -27,7 +27,6 @@ uiModules.get('apps/management')
         { title: 'format' },
         { title: 'searchable', info: 'These fields can be used in the filter bar' },
         { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
-        { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
         { title: 'excluded', info: 'Fields that are excluded from _source when it is fetched' },
         { title: 'controls', sortable: false }
       ];
@@ -77,10 +76,6 @@ uiModules.get('apps/management')
               markup: field.aggregatable ? yesTemplate : noTemplate,
               value: field.aggregatable
             },
-            {
-              markup: field.analyzed ? yesTemplate : noTemplate,
-              value: field.analyzed
-            },
             {
               markup: excluded ? yesTemplate : noTemplate,
               value: excluded

diff --git a/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js
@@ -1,5 +1,7 @@
 import _ from 'lodash';
+
 import handleESError from '../../../lib/handle_es_error';
+import { shouldReadFieldFromDocValues } from './should_read_field_from_doc_values';
 
 export function registerFieldCapabilities(server) {
   server.route({
@@ -19,7 +21,11 @@ export function registerFieldCapabilities(server) {
         (res) => {
           const fields = _.get(res, 'indices._all.fields', {});
           const fieldsFilteredValues = _.mapValues(fields, (value) => {
-            return _.pick(value, ['searchable', 'aggregatable']);
+            return {
+              searchable: value.searchable,
+              aggregatable: value.aggregatable,
+              readFromDocValues: shouldReadFieldFromDocValues(value.aggregatable, value.type)
+            };
           });
 
           const retVal = { fields: fieldsFilteredValues };

diff --git a/src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js b/src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js
@@ -0,0 +1,4 @@
+// see https://github.com/elastic/kibana/issues/11141 for details
+export function shouldReadFieldFromDocValues(aggregatable, esType) {
+  return aggregatable && esType !== 'text' && !esType.startsWith('_');
+}
diff --git a/src/fixtures/logstash_fields.js b/src/fixtures/logstash_fields.js
@@ -1,52 +1,48 @@
 import { castEsToKbnFieldTypeName } from '../utils';
+import { shouldReadFieldFromDocValues } from '../core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values';
 
 function stubbedLogstashFields() {
   return [
-    //                                  |indexed
-    //                                  |      |analyzed
-    //                                  |      |      |aggregatable
-    //                                  |      |      |      |searchable
-    // name               esType        |      |      |      |     |metadata
-    ['bytes',             'long',       true,  true,  true,  true,  { count: 10, docValues: true } ],
-    ['ssl',               'boolean',    true,  true,  true,  true,  { count: 20 } ],
-    ['@timestamp',        'date',       true,  true,  true,  true,  { count: 30 } ],
-    ['time',              'date',       true,  true,  true,  true,  { count: 30 } ],
-    ['@tags',             'keyword',    true,  true,  true,  true ],
-    ['utc_time',          'date',       true,  true,  true,  true ],
-    ['phpmemory',         'integer',    true,  true,  true,  true ],
-    ['ip',                'ip',         true,  true,  true,  true ],
-    ['request_body',      'attachment', true,  true,  true,  true ],
-    ['point',             'geo_point',  true,  true,  true,  true ],
-    ['area',              'geo_shape',  true,  true,  true,  true ],
-    ['hashed',            'murmur3',    true,  true,  false, true ],
-    ['geo.coordinates',   'geo_point',  true,  true,  true,  true ],
-    ['extension',         'keyword',    true,  true,  true,  true ],
-    ['machine.os',        'text',       true,  true,  true,  true ],
-    ['machine.os.raw',    'keyword',    true,  false, true,  true,  { docValues: true } ],
-    ['geo.src',           'keyword',    true,  true,  true,  true ],
-    ['_id',               'keyword',    false, false, true,  true ],
-    ['_type',             'keyword',    false, false, true,  true ],
-    ['_source',           'keyword',    false, false, true,  true ],
-    ['non-filterable',    'text',       false, false, true,  false],
-    ['non-sortable',      'text',       false, false, false, false],
-    ['custom_user_field', 'conflict',   false, false, true,  true ],
-    ['script string',     'text',       false, false, true,  false, { script: '\'i am a string\'' } ],
-    ['script number',     'long',       false, false, true,  false, { script: '1234' } ],
-    ['script date',       'date',       false, false, true,  false, { script: '1234', lang: 'painless' } ],
-    ['script murmur3',    'murmur3',    false, false, true,  false, { script: '1234' } ],
+    //                                  |aggregatable
+    //                                  |      |searchable
+    // name               esType        |      |     |metadata
+    ['bytes',             'long',       true,  true,  { count: 10 } ],
+    ['ssl',               'boolean',    true,  true,  { count: 20 } ],
+    ['@timestamp',        'date',       true,  true,  { count: 30 } ],
+    ['time',              'date',       true,  true,  { count: 30 } ],
+    ['@tags',             'keyword',    true,  true ],
+    ['utc_time',          'date',       true,  true ],
+    ['phpmemory',         'integer',    true,  true ],
+    ['ip',                'ip',         true,  true ],
+    ['request_body',      'attachment', true,  true ],
+    ['point',             'geo_point',  true,  true ],
+    ['area',              'geo_shape',  true,  true ],
+    ['hashed',            'murmur3',    false, true ],
+    ['geo.coordinates',   'geo_point',  true,  true ],
+    ['extension',         'keyword',    true,  true ],
+    ['machine.os',        'text',       true,  true ],
+    ['machine.os.raw',    'keyword',    true,  true ],
+    ['geo.src',           'keyword',    true,  true ],
+    ['_id',               '_id',        true,  true ],
+    ['_type',             '_type',      true,  true ],
+    ['_source',           '_source',    true,  true ],
+    ['non-filterable',    'text',       true,  false],
+    ['non-sortable',      'text',       false, false],
+    ['custom_user_field', 'conflict',   true,  true ],
+    ['script string',     'text',       true,  false, { script: '\'i am a string\'' } ],
+    ['script number',     'long',       true,  false, { script: '1234' } ],
+    ['script date',       'date',       true,  false, { script: '1234', lang: 'painless' } ],
+    ['script murmur3',    'murmur3',    true,  false, { script: '1234' } ],
   ].map(function (row) {
     const [
       name,
       esType,
-      indexed,
-      analyzed,
       aggregatable,
       searchable,
       metadata = {}
     ] = row;
 
     const {
-      docValues = false,
       count = 0,
       script,
       lang = script ? 'expression' : undefined,
@@ -60,9 +56,7 @@ function stubbedLogstashFields() {
     return {
       name,
       type,
-      doc_values: docValues,
-      indexed,
-      analyzed,
+      readFromDocValues: shouldReadFieldFromDocValues(aggregatable, esType),
       aggregatable,
       searchable,
       count,

diff --git a/src/ui/public/agg_types/__tests__/metrics/top_hit.js b/src/ui/public/agg_types/__tests__/metrics/top_hit.js
@@ -71,15 +71,27 @@ describe('Top hit metric', function () {
     expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
   });
 
-  it('should request both for the source and doc_values fields', function () {
+  it('requests both source and docvalues_fields for non-text aggregatable fields', function () {
     init({ field: 'bytes' });
     expect(aggDsl.top_hits._source).to.be('bytes');
     expect(aggDsl.top_hits.docvalue_fields).to.eql([ 'bytes' ]);
   });
 
-  it('should only request for the source if the field does not have the doc_values property', function () {
-    init({ field: 'ssl' });
-    expect(aggDsl.top_hits._source).to.be('ssl');
+  it('requests just source for aggregatable text fields', function () {
+    init({ field: 'machine.os' });
+    expect(aggDsl.top_hits._source).to.be('machine.os');
+    expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
+  });
+
+  it('requests just source for not-aggregatable text fields', function () {
+    init({ field: 'non-sortable' });
+    expect(aggDsl.top_hits._source).to.be('non-sortable');
+    expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
+  });
+
+  it('requests just source for not-aggregatable, non-text fields', function () {
+    init({ field: 'hashed' });
+    expect(aggDsl.top_hits._source).to.be('hashed');
     expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
   });
 

diff --git a/src/ui/public/agg_types/controls/field.html b/src/ui/public/agg_types/controls/field.html
@@ -3,25 +3,6 @@
     Field
   </label>
 
-  <span class="pull-right text-warning hintbox-label" ng-show="aggParam.showAnalyzedWarning && agg.params.field.analyzed"
-      ng-click="showAnalyzedFieldWarning = !showAnalyzedFieldWarning">
-    <i class="fa fa-warning"></i> Analyzed Field
-  </span>
-  <div class="hintbox" ng-show="showAnalyzedFieldWarning && agg.params.field.analyzed">
-    <p>
-    <strong>Careful!</strong> The field selected contains analyzed strings. Analyzed strings are highly unique and can use a lot of memory to visualize. Values such as <i>foo-bar</i> will be broken into <i>foo</i> and <i>bar</i>. See <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html" target="_blank">Mapping Types</a> for more information on setting this field as <i>keyword</i> (not analyzed).
-    </p>
-
-    <p ng-show="indexedFields.byName[agg.params.field.name + '.keyword'].analyzed == false">
-      <strong>Tip:</strong> <i>{{agg.params.field.name + '.keyword'}}</i> may be a non analyzed version of this field.
-    </p>
-
-    <p ng-show="indexedFields.byName[agg.params.field.name + '.raw'].analyzed == false">
-      <strong>Tip:</strong> <i>{{agg.params.field.name + '.raw'}}</i> may be a non analyzed version of this field.
-    </p>
-  </div>
-
-
   <ui-select
     name="field"
     required

diff --git a/src/ui/public/agg_types/metrics/top_hit.js b/src/ui/public/agg_types/metrics/top_hit.js
@@ -24,7 +24,6 @@ export function AggTypesMetricsTopHitProvider(Private) {
       {
         name: 'field',
         onlyAggregatable: false,
-        showAnalyzedWarning: false,
         filterFieldTypes: function (vis, value) {
           if (vis.type.name === 'table' || vis.type.name === 'metric') {
             return true;
@@ -45,7 +44,7 @@ export function AggTypesMetricsTopHitProvider(Private) {
               }
             };
           } else {
-            if (field.doc_values) {
+            if (field.readFromDocValues) {
               output.params.docvalue_fields = [ field.name ];
             }
             output.params._source = field.name === '_source' ? true : field.name;

diff --git a/src/ui/public/agg_types/param_types/field.js b/src/ui/public/agg_types/param_types/field.js
@@ -20,8 +20,6 @@ export function AggTypesParamTypesFieldProvider(Private, $filter) {
   FieldAggParam.prototype.filterFieldTypes = '*';
   // retain only the fields with the aggregatable property if the onlyAggregatable option is true
   FieldAggParam.prototype.onlyAggregatable = true;
-  // show a warning about the field being analyzed
-  FieldAggParam.prototype.showAnalyzedWarning = true;
 
   /**
    * Called to serialize values for saving an aggConfig object

diff --git a/src/ui/public/index_patterns/__tests__/_index_pattern.js b/src/ui/public/index_patterns/__tests__/_index_pattern.js
@@ -157,7 +157,8 @@ describe('index pattern', function () {
         sortable: true,
         type: 'number',
         aggregatable: true,
-        searchable: false
+        searchable: false,
+        readFromDocValues: true
       },
       {
         name: 'script number',

diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js
@@ -42,6 +42,7 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
     const scripted = !!spec.scripted;
     const searchable = !!spec.searchable || scripted;
     const aggregatable = !!spec.aggregatable || scripted;
+    const readFromDocValues = !!spec.readFromDocValues && !scripted;
     const sortable = spec.name === '_score' || ((indexed || aggregatable) && type.sortable);
     const filterable = spec.name === '_id' || scripted || ((indexed || searchable) && type.filterable);
     const visualizable = aggregatable;
@@ -55,14 +56,10 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
     obj.writ('script', scripted ? spec.script : null);
     obj.writ('lang', scripted ? (spec.lang || 'painless') : null);
 
-    // mapping info
-    obj.fact('indexed', indexed);
-    obj.fact('analyzed', !!spec.analyzed);
-    obj.fact('doc_values', !!spec.doc_values);
-
     // stats
     obj.fact('searchable', searchable);
     obj.fact('aggregatable', aggregatable);
+    obj.fact('readFromDocValues', readFromDocValues);
 
     // usage flags, read-only and won't be saved
     obj.comp('format', format);
@@ -81,6 +78,24 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
     return obj.create();
   }
 
+  Object.defineProperties(Field.prototype, {
+    indexed: {
+      get() {
+        throw new Error('field.indexed has been removed, see https://github.com/elastic/kibana/pull/11969');
+      }
+    },
+    analyzed: {
+      get() {
+        throw new Error('field.analyzed has been removed, see https://github.com/elastic/kibana/pull/11969');
+      }
+    },
+    doc_values: {
+      get() {
+        throw new Error('field.doc_values has been removed, see https://github.com/elastic/kibana/pull/11969');
+      }
+    },
+  });
+
   Field.prototype.routes = {
     edit: '/management/kibana/indices/{{indexPattern.id}}/field/{{name}}'
   };