diff --git a/package.json b/package.json index 9249600018f1c..5f620959a5707 100644 --- a/package.json +++ b/package.json @@ -132,6 +132,7 @@ "moment": "2.10.6", "moment-timezone": "0.4.1", "node-uuid": "1.4.7", + "papaparse": "4.1.2", "raw-loader": "0.5.1", "request": "2.61.0", "rimraf": "2.4.3", diff --git a/src/plugins/kibana/public/settings/sections/indices/_create.html b/src/plugins/kibana/public/settings/sections/indices/_create.html index 7cf592c694419..fa65016ff9f26 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_create.html +++ b/src/plugins/kibana/public/settings/sections/indices/_create.html @@ -62,6 +62,7 @@

Time-interval based index patterns are deprecated!

ng-attr-placeholder="{{index.defaultName}}" ng-model-options="{ updateOn: 'default blur', debounce: {'default': 2500, 'blur': 0} }" validate-index-name + allow-wildcard name="name" required type="text" diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.html b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.html new file mode 100644 index 0000000000000..8c0a0d7812b4c --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.html @@ -0,0 +1,63 @@ + +

Pick a CSV file to get started. + Please follow the instructions below. +

+ +
+
Drop your file here
+
or
+ +
Maximum upload file size: 1 GB
+
+ + +
+

Review the sample below. + Click next if it looks like we parsed your file correctly. +

+ +
+ +
+ +
+ +
+ +
+ + + + + + + {{ wizard.file.name }} + +
+ +
+ + + + + + + + + + + +
+ {{ col | limitTo:12 }}{{ col.length > 12 ? '...' : '' }} +
{{ cell }}
+
+
diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.js b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.js new file mode 100644 index 0000000000000..7699cb1d89e72 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.js @@ -0,0 +1,132 @@ +import _ from 'lodash'; +import Papa from 'papaparse'; +import modules from 'ui/modules'; +import template from './parse_csv_step.html'; +import './styles/_add_data_parse_csv_step.less'; + +modules.get('apps/settings') + .directive('parseCsvStep', function () { + return { + restrict: 'E', + template: template, + scope: { + file: '=', + parseOptions: '=', + samples: '=' + }, + bindToController: true, + controllerAs: 'wizard', + controller: function ($scope) { + const maxSampleRows = 10; + const maxSampleColumns = 20; + + this.delimiterOptions = [ + { + label: 'comma', + value: ',' + }, + { + label: 'tab', + value: '\t' + }, + { + label: 'space', + value: ' ' + }, + { + label: 'semicolon', + value: ';' + }, + { + label: 'pipe', + value: '|' + } + ]; + + this.parse = () => { + if (!this.file) return; + let row = 1; + let rows = []; + let data = []; + + const config = _.assign( + { + header: true, + dynamicTyping: true, + step: (results, parser) => { + if (row > maxSampleRows) { + parser.abort(); + + // The complete callback isn't automatically called if parsing is manually aborted + config.complete(); + return; + } + if (row === 1) { + // Collect general information on the first pass + if (results.meta.fields.length > _.uniq(results.meta.fields).length) { + this.formattedErrors.push('Column names must be unique'); + } + _.forEach(results.meta.fields, (field) => { + if (_.isEmpty(field)) { + this.formattedErrors.push('Column names must not be blank'); + } + }); + if (results.meta.fields.length > maxSampleColumns) { + this.formattedWarnings.push(`Preview truncated to ${maxSampleColumns} columns`); + } + + this.columns = results.meta.fields.slice(0, maxSampleColumns); + this.parseOptions = _.defaults({}, this.parseOptions, {delimiter: results.meta.delimiter}); + } + + this.formattedErrors = _.map(results.errors, (error) => { + return `${error.type} at line ${row + 1} - ${error.message}`; + }); + + data = data.concat(results.data); + + rows = rows.concat(_.map(results.data, (row) => { + return _.map(this.columns, (columnName) => { + return row[columnName]; + }); + })); + + ++row; + }, + complete: () => { + $scope.$apply(() => { + this.rows = rows; + + if (_.isUndefined(this.formattedErrors) || _.isEmpty(this.formattedErrors)) { + this.samples = data; + } + else { + delete this.samples; + } + }); + } + }, + this.parseOptions + ); + + Papa.parse(this.file, config); + }; + + $scope.$watch('wizard.parseOptions', (newValue, oldValue) => { + // Delimiter is auto-detected in the first run of the parse function, so we don't want to + // re-parse just because it's being initialized. + if (!_.isUndefined(oldValue)) { + this.parse(); + } + }, true); + + $scope.$watch('wizard.file', () => { + delete this.rows; + delete this.columns; + delete this.formattedErrors; + this.formattedWarnings = []; + this.parse(); + }); + } + }; + }); diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/styles/_add_data_parse_csv_step.less b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/styles/_add_data_parse_csv_step.less new file mode 100644 index 0000000000000..036c4f940e432 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/parse_csv_step/styles/_add_data_parse_csv_step.less @@ -0,0 +1,64 @@ +@import (reference) "../../../styles/_add_data_wizard"; + +.upload-wizard-file-upload-container { + min-height: 300px; + display: flex; + flex-direction: column; + justify-content: center; + background-color: @settings-add-data-wizard-form-control-bg; + border: @settings-add-data-wizard-parse-csv-container-border 1px dashed; + text-align: center; + + .upload-instructions { + font-size: 2em; + } + + .upload-instructions-separator { + margin: 15px 0; + } + + button { + width: inherit; + } + + button.upload { + align-self: center; + margin-bottom: 15px; + } +} + +.upload-wizard-file-preview-container { + .preview { + overflow: auto; + max-height: 500px; + border: @settings-add-data-wizard-parse-csv-container-border 1px solid; + + table { + margin-bottom: 0; + + .table-striped() + } + } + + .parse-error { + margin-top: 2em; + } + + .advanced-options { + display: flex; + align-items: center; + + .form-group { + display: flex; + align-items: center; + padding-right: 15px; + + label { + padding-right: 8px; + margin-bottom: 0; + } + } + + padding-bottom: 10px; + } +} diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.html b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.html index e10e8bf1d690d..4816d7c9ac0f3 100644 --- a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.html +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.html @@ -3,31 +3,48 @@

Review the index pattern. fields can be changed if we got it wrong!

-
-
-
{{ error }}
-
- - Patterns allow you to define dynamic index names using * as a wildcard. Example: filebeat-* - - - -
+
+
+
+
{{ error }}
+
+
+ Index names must be all lowercase +
+
+ An index name must not be empty and cannot contain whitespace or any of the following characters: ", *, \, <, |, ,, >, /, ? +
- - + + {{ reviewStep.patternInput.helpText }} + + + +
+ + +
diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.js b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.js index b607c0b22026e..75eedc0624d68 100644 --- a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.js +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/pattern_review_step/pattern_review_step.js @@ -6,6 +6,7 @@ import isGeoPointObject from './lib/is_geo_point_object'; import forEachField from './lib/for_each_field'; import './styles/_add_data_pattern_review_step.less'; import moment from 'moment'; +import '../../../../../../../../ui/public/directives/validate_lowercase'; function pickDefaultTimeFieldName(dateFields) { if (_.isEmpty(dateFields)) { @@ -26,7 +27,8 @@ modules.get('apps/settings') scope: { indexPattern: '=', pipeline: '=', - sampleDoc: '=' + sampleDoc: '=', + defaultIndexInput: '=' }, controllerAs: 'reviewStep', bindToController: true, @@ -34,6 +36,17 @@ modules.get('apps/settings') this.errors = []; const sampleFields = {}; + this.patternInput = { + label: 'Index name', + helpText: 'The name of the Elasticsearch index you want to create for your data.', + defaultValue: '', + placeholder: 'Name' + }; + + if (this.defaultIndexInput) { + this.patternInput.defaultValue = this.defaultIndexInput; + } + if (_.isUndefined(this.indexPattern)) { this.indexPattern = {}; } @@ -62,7 +75,7 @@ modules.get('apps/settings') }); _.defaults(this.indexPattern, { - id: 'filebeat-*', + id: this.patternInput.defaultValue, title: 'filebeat-*', fields: _(sampleFields) .map((field, fieldName) => { diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/styles/_add_data_upload_data_step.less b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/styles/_add_data_upload_data_step.less new file mode 100644 index 0000000000000..e8e472de50c8f --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/styles/_add_data_upload_data_step.less @@ -0,0 +1,22 @@ +@import (reference) "../../../styles/_add_data_wizard"; + +@add-data-upload-step-multi-alert-padding: 2px; + +.bulk-results { + .alert-warning { + padding: @add-data-upload-step-multi-alert-padding; + } + + ul.errors { + background-color: white; + color: @text-color; + padding: @alert-padding - @add-data-upload-step-multi-alert-padding; + list-style-position: inside; + } + + .alert-title { + display: flex; + padding: @alert-padding - @add-data-upload-step-multi-alert-padding; + justify-content: space-between; + } +} diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.html b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.html new file mode 100644 index 0000000000000..d06bf536d0b5f --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.html @@ -0,0 +1,29 @@ +
+

Sit back, relax, we'll take it from here.

+ +
+ We're loading your data now. This may take some time if you selected a large file. +
+
+ + +
+

Upload complete. Let's take a look:

+ +
+ Created {{ uploadStep.created }} documents!
+
+
+
+ We encountered errors while indexing your data + + {{uploadStep.showAllErrors ? "Show Less" : "Show More"}} + +
+ +
+
diff --git a/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.js b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.js new file mode 100644 index 0000000000000..93317c9134476 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/add_data_steps/upload_data_step/upload_data_step.js @@ -0,0 +1,65 @@ +import modules from 'ui/modules'; +import template from './upload_data_step.html'; +import _ from 'lodash'; +import IngestProvider from 'ui/ingest'; +import './styles/_add_data_upload_data_step.less'; + +function formatIndexError(errorDoc) { + const lineNumber = errorDoc._id.substr(errorDoc._id.lastIndexOf(':') + 1); + const errorType = errorDoc.error.type; + const errorReason = errorDoc.error.reason; + + return `Line ${lineNumber}: ${errorType} - ${errorReason}`; +} + +modules.get('apps/settings') +.directive('uploadDataStep', function () { + return { + template: template, + scope: { + results: '=' + }, + bindToController: true, + controllerAs: 'uploadStep', + controller: function (Notifier, $window, Private, $scope) { + const ingest = Private(IngestProvider); + const notify = new Notifier({ + location: 'Add Data' + }); + + const usePipeline = !_.isEmpty(_.get(this.results, 'pipeline.processors')); + ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter, usePipeline) + .then( + (res) => { + this.created = 0; + this.formattedErrors = []; + _.forEach(res.data, (response) => { + this.created += response.created; + this.formattedErrors = this.formattedErrors.concat(_.map(_.get(response, 'errors.index'), formatIndexError)); + if (!_.isEmpty(_.get(response, 'errors.other'))) { + this.formattedErrors = this.formattedErrors.concat(response.errors.other); + } + }); + }, + (err) => { + notify.error(err); + $window.scrollTo(0, 0); + } + ); + + this.showAllErrors = false; + this.defaultErrorLimit = 10; + this.displayErrors = []; + $scope.$watchGroup(['uploadStep.formattedErrors', 'uploadStep.showAllErrors'], (newValues) => { + const [formattedErrors, showAllErrors] = newValues; + + if (showAllErrors && formattedErrors) { + this.displayErrors = formattedErrors; + } + else if (formattedErrors) { + this.displayErrors = formattedErrors.slice(0, this.defaultErrorLimit + 1); + } + }); + } + }; +}); diff --git a/src/plugins/kibana/public/settings/sections/indices/filebeat/directives/filebeat_wizard.js b/src/plugins/kibana/public/settings/sections/indices/filebeat/directives/filebeat_wizard.js index 6de8287171bff..315bb94182ff0 100644 --- a/src/plugins/kibana/public/settings/sections/indices/filebeat/directives/filebeat_wizard.js +++ b/src/plugins/kibana/public/settings/sections/indices/filebeat/directives/filebeat_wizard.js @@ -16,7 +16,7 @@ modules.get('apps/settings') scope: {}, bindToController: true, controllerAs: 'wizard', - controller: function ($scope, AppState, safeConfirm, kbnUrl, $http, Notifier, $window, config, Private) { + controller: function ($scope, AppState, safeConfirm, kbnUrl, Notifier, $window, Private) { const ingest = Private(IngestProvider); const $state = this.state = new AppState(); diff --git a/src/plugins/kibana/public/settings/sections/indices/index.html b/src/plugins/kibana/public/settings/sections/indices/index.html index a7a1c40b6c4d7..2fec61cf7d747 100644 --- a/src/plugins/kibana/public/settings/sections/indices/index.html +++ b/src/plugins/kibana/public/settings/sections/indices/index.html @@ -13,6 +13,13 @@

Pick this option if you already have data in Elasticsearch.
+ +

+ Upload +

+
+ Got CSVs? Upload them here. No pain, all gain. +
diff --git a/src/plugins/kibana/public/settings/sections/indices/index.js b/src/plugins/kibana/public/settings/sections/indices/index.js index e9b0012ace707..0d4c1f58a1b30 100644 --- a/src/plugins/kibana/public/settings/sections/indices/index.js +++ b/src/plugins/kibana/public/settings/sections/indices/index.js @@ -5,6 +5,7 @@ import 'plugins/kibana/settings/sections/indices/_create'; import 'plugins/kibana/settings/sections/indices/_edit'; import 'plugins/kibana/settings/sections/indices/_field_editor'; import 'plugins/kibana/settings/sections/indices/filebeat/index'; +import 'plugins/kibana/settings/sections/indices/upload/index'; import uiRoutes from 'ui/routes'; import uiModules from 'ui/modules'; import indexTemplate from 'plugins/kibana/settings/sections/indices/index.html'; diff --git a/src/plugins/kibana/public/settings/sections/indices/styles/_add_data_wizard.less b/src/plugins/kibana/public/settings/sections/indices/styles/_add_data_wizard.less index e8354811ae026..c0ebea3e41866 100644 --- a/src/plugins/kibana/public/settings/sections/indices/styles/_add_data_wizard.less +++ b/src/plugins/kibana/public/settings/sections/indices/styles/_add_data_wizard.less @@ -52,6 +52,11 @@ } } + .btn-lg { + padding: 6px 35px; + font-size: 1.2em; + } + .form-group { margin-bottom: 5px; } diff --git a/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.html b/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.html new file mode 100644 index 0000000000000..a36d8eb4fd055 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.html @@ -0,0 +1,90 @@ +
+
+ + 1. Select + + + 2. Review + + + 3. Upload + +
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+ + + +
+
+ +
+
+ +
+
+
+
+ +
+ + +
+
+
+ +
+
+
+
+
+
diff --git a/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.js b/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.js new file mode 100644 index 0000000000000..a485ba28ce39b --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/upload/directives/upload_wizard.js @@ -0,0 +1,95 @@ +import modules from 'ui/modules'; +import template from 'plugins/kibana/settings/sections/indices/upload/directives/upload_wizard.html'; +import IngestProvider from 'ui/ingest'; +import 'plugins/kibana/settings/sections/indices/add_data_steps/pattern_review_step'; +import 'plugins/kibana/settings/sections/indices/add_data_steps/parse_csv_step'; +import 'plugins/kibana/settings/sections/indices/add_data_steps/upload_data_step'; +import '../../styles/_add_data_wizard.less'; + +modules.get('apps/settings') + .directive('uploadWizard', function () { + return { + restrict: 'E', + template: template, + scope: {}, + bindToController: true, + controllerAs: 'wizard', + controller: function ($scope, AppState, safeConfirm, kbnUrl, Notifier, $window, Private) { + const ingest = Private(IngestProvider); + const $state = this.state = new AppState(); + + var notify = new Notifier({ + location: 'Add Data' + }); + + var totalSteps = 3; + this.stepResults = {}; + + this.setCurrentStep = (step) => { + if (!this.complete) { + $state.currentStep = step; + $state.save(); + } + }; + this.setCurrentStep(0); + + this.nextStep = () => { + if ($state.currentStep + 1 < totalSteps) { + this.setCurrentStep($state.currentStep + 1); + } + else if ($state.currentStep + 1 === totalSteps) { + kbnUrl.change('/discover', null, {index: this.stepResults.indexPattern.id}); + } + }; + + this.prevStep = () => { + if ($state.currentStep > 0) { + this.setCurrentStep($state.currentStep - 1); + } + }; + + this.save = () => { + return ingest.save(this.stepResults.indexPattern) + .then( + () => { + this.nextStep(); + }, + (err) => { + notify.error(err); + $window.scrollTo(0,0); + } + ); + }; + + $scope.$watch('wizard.state.currentStep', (newValue, oldValue) => { + if (this.complete) { + $state.currentStep = totalSteps - 1; + $state.save(); + return; + } + if (newValue + 1 === totalSteps) { + this.complete = true; + } + if (newValue < oldValue) { + return safeConfirm('Going back will reset any changes you\'ve made to this step, do you want to continue?') + .then( + () => { + if ($state.currentStep < 1) { + delete this.stepResults.indexPattern; + } + this.currentStep = newValue; + }, + () => { + $state.currentStep = oldValue; + $state.save(); + } + ); + } + else { + this.currentStep = newValue; + } + }); + } + }; + }); + diff --git a/src/plugins/kibana/public/settings/sections/indices/upload/index.html b/src/plugins/kibana/public/settings/sections/indices/upload/index.html new file mode 100644 index 0000000000000..123fc4b91b114 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/upload/index.html @@ -0,0 +1,3 @@ + + + diff --git a/src/plugins/kibana/public/settings/sections/indices/upload/index.js b/src/plugins/kibana/public/settings/sections/indices/upload/index.js new file mode 100644 index 0000000000000..473dac7e7eb48 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/upload/index.js @@ -0,0 +1,7 @@ +import routes from 'ui/routes'; +import template from 'plugins/kibana/settings/sections/indices/upload/index.html'; +import './directives/upload_wizard'; + +routes.when('/settings/indices/create/upload', { + template: template +}); diff --git a/src/plugins/kibana/public/settings/styles/main.less b/src/plugins/kibana/public/settings/styles/main.less index 002ea4c2f0852..ab760fb42f87d 100644 --- a/src/plugins/kibana/public/settings/styles/main.less +++ b/src/plugins/kibana/public/settings/styles/main.less @@ -204,4 +204,3 @@ kbn-settings-indices { .kbn-settings-indices-create { .time-and-pattern > div {} } - diff --git a/src/ui/public/directives/__tests__/validate_index_name.js b/src/ui/public/directives/__tests__/validate_index_name.js index f792f65c8174a..3deb454077e1e 100644 --- a/src/ui/public/directives/__tests__/validate_index_name.js +++ b/src/ui/public/directives/__tests__/validate_index_name.js @@ -8,7 +8,8 @@ import 'ui/directives/validate_index_name'; describe('Validate index name directive', function () { let $compile; let $rootScope; - let html = ''; + let noWildcardHtml = ''; + let allowWildcardHtml = ''; beforeEach(ngMock.module('kibana')); @@ -17,14 +18,14 @@ describe('Validate index name directive', function () { $rootScope = _$rootScope_; })); - function checkPattern(input) { + function checkPattern(input, html) { $rootScope.indexName = input; let element = $compile(html)($rootScope); $rootScope.$digest(); return element; } - let badPatterns = [ + const badPatterns = [ null, undefined, '', @@ -41,19 +42,22 @@ describe('Validate index name directive', function () { 'foo,bar', ]; - let goodPatterns = [ + const goodPatterns = [ '...', 'foo', 'foo.bar', + '[foo-]YYYY-MM-DD', + ]; + + const wildcardPatterns = [ 'foo*', 'foo.bar*', - 'foo.*', - '[foo-]YYYY-MM-DD', + 'foo.*' ]; badPatterns.forEach(function (pattern) { it('should not accept index pattern: ' + pattern, function () { - let element = checkPattern(pattern); + let element = checkPattern(pattern, noWildcardHtml); expect(element.hasClass('ng-invalid')).to.be(true); expect(element.hasClass('ng-valid')).to.not.be(true); }); @@ -61,7 +65,23 @@ describe('Validate index name directive', function () { goodPatterns.forEach(function (pattern) { it('should accept index pattern: ' + pattern, function () { - let element = checkPattern(pattern); + let element = checkPattern(pattern, noWildcardHtml); + expect(element.hasClass('ng-invalid')).to.not.be(true); + expect(element.hasClass('ng-valid')).to.be(true); + }); + }); + + it('should disallow wildcards by default', function () { + wildcardPatterns.forEach(function (pattern) { + let element = checkPattern(pattern, noWildcardHtml); + expect(element.hasClass('ng-invalid')).to.be(true); + expect(element.hasClass('ng-valid')).to.not.be(true); + }); + }); + + it('should allow wildcards if the allow-wildcard attribute is present', function () { + wildcardPatterns.forEach(function (pattern) { + let element = checkPattern(pattern, allowWildcardHtml); expect(element.hasClass('ng-invalid')).to.not.be(true); expect(element.hasClass('ng-valid')).to.be(true); }); diff --git a/src/ui/public/directives/file_upload.js b/src/ui/public/directives/file_upload.js index 03fb7137806d8..06c4752b456bf 100644 --- a/src/ui/public/directives/file_upload.js +++ b/src/ui/public/directives/file_upload.js @@ -22,7 +22,7 @@ module.directive('fileUpload', function () { const handleFile = (file) => { if (_.isUndefined(file)) return; - if ($scope.onRead) { + if (_.has(attrs, 'onRead')) { let reader = new FileReader(); reader.onload = function (e) { $scope.$apply(function () { @@ -32,8 +32,10 @@ module.directive('fileUpload', function () { reader.readAsText(file); } - if ($scope.onLocate) { - $scope.onLocate({ file }); + if (_.has(attrs, 'onLocate')) { + $scope.$apply(function () { + $scope.onLocate({ file }); + }); } }; diff --git a/src/ui/public/directives/validate_index_name.js b/src/ui/public/directives/validate_index_name.js index 1082616ee0e7c..10b51c4dfb8fb 100644 --- a/src/ui/public/directives/validate_index_name.js +++ b/src/ui/public/directives/validate_index_name.js @@ -8,11 +8,13 @@ uiModules return { restrict: 'A', require: 'ngModel', - scope: { - 'ngModel': '=' - }, link: function ($scope, elem, attr, ngModel) { - let illegalCharacters = ['\\', '/', '?', '"', '<', '>', '|', ' ', ',']; + const illegalCharacters = ['\\', '/', '?', '"', '<', '>', '|', ' ', ',']; + const allowWildcard = !_.isUndefined(attr.allowWildcard) && attr.allowWildcard !== 'false'; + if (!allowWildcard) { + illegalCharacters.push('*'); + } + let isValid = function (input) { if (input == null || input === '' || input === '.' || input === '..') return false; @@ -22,19 +24,9 @@ uiModules return !match; }; - // From User - ngModel.$parsers.unshift(function (value) { - let valid = isValid(value); - ngModel.$setValidity('indexNameInput', valid); - return valid ? value : undefined; - }); - - // To user - ngModel.$formatters.unshift(function (value) { - ngModel.$setValidity('indexNameInput', isValid(value)); - return value; - }); - + ngModel.$validators.indexNameInput = function (modelValue, viewValue) { + return isValid(viewValue); + }; } }; }); diff --git a/src/ui/public/directives/validate_lowercase.js b/src/ui/public/directives/validate_lowercase.js new file mode 100644 index 0000000000000..fb37841ee6319 --- /dev/null +++ b/src/ui/public/directives/validate_lowercase.js @@ -0,0 +1,21 @@ +import uiModules from 'ui/modules'; + +uiModules +.get('kibana') +.directive('validateLowercase', function () { + return { + restrict: 'A', + require: 'ngModel', + link: function ($scope, elem, attr, ctrl) { + ctrl.$validators.lowercase = function (modelValue, viewValue) { + if (ctrl.$isEmpty(modelValue)) { + // consider empty models to be valid per lowercase rules + return true; + } + + return viewValue.toLowerCase() === viewValue; + }; + } + }; +}); + diff --git a/src/ui/public/ingest/__tests__/ingest.js b/src/ui/public/ingest/__tests__/ingest.js index 16dbb7ebf1c4b..4ca53e5e9db3f 100644 --- a/src/ui/public/ingest/__tests__/ingest.js +++ b/src/ui/public/ingest/__tests__/ingest.js @@ -26,7 +26,7 @@ describe('Ingest Service', function () { it('Sets the default index if there isn\'t one already', function () { $httpBackend - .when('POST', '../api/kibana/ingest') + .when('POST', '/api/kibana/ingest') .respond('ok'); expect(config.get('defaultIndex')).to.be(null); @@ -38,7 +38,7 @@ describe('Ingest Service', function () { it('Returns error from ingest API if there is one', function (done) { $httpBackend - .expectPOST('../api/kibana/ingest') + .expectPOST('/api/kibana/ingest') .respond(400); ingest.save({id: 'foo'}) @@ -57,7 +57,7 @@ describe('Ingest Service', function () { it('Broadcasts an ingest:updated event on the rootScope upon succesful save', function () { $httpBackend - .when('POST', '../api/kibana/ingest') + .when('POST', '/api/kibana/ingest') .respond('ok'); ingest.save({id: 'foo'}); @@ -75,7 +75,7 @@ describe('Ingest Service', function () { it('Calls the DELETE endpoint of the ingest API with the given id', function () { $httpBackend - .expectDELETE('../api/kibana/ingest/foo') + .expectDELETE('/api/kibana/ingest/foo') .respond('ok'); ingest.delete('foo'); @@ -84,7 +84,7 @@ describe('Ingest Service', function () { it('Returns error from ingest API if there is one', function (done) { $httpBackend - .expectDELETE('../api/kibana/ingest/foo') + .expectDELETE('/api/kibana/ingest/foo') .respond(404); ingest.delete('foo') @@ -103,7 +103,7 @@ describe('Ingest Service', function () { it('Broadcasts an ingest:updated event on the rootScope upon succesful save', function () { $httpBackend - .when('DELETE', '../api/kibana/ingest/foo') + .when('DELETE', '/api/kibana/ingest/foo') .respond('ok'); ingest.delete('foo'); @@ -114,11 +114,53 @@ describe('Ingest Service', function () { }); }); + describe('uploadCSV', function () { + it('throws an error if file and index pattern are not provided', function () { + expect(ingest.uploadCSV).to.throwException(/file is required/); + expect(ingest.uploadCSV).withArgs('foo').to.throwException(/index pattern is required/); + }); + + it('POSTs to the kibana _data endpoint with the correct params and the file attached as multipart/form-data', function () { + $httpBackend + .expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true', function (data) { + // The assertions we can do here are limited because of poor browser support for FormData methods + return data instanceof FormData; + }) + .respond('ok'); + + const file = new Blob(['foo,bar'], {type : 'text/csv'}); + + ingest.uploadCSV(file, 'foo', ';', true); + $httpBackend.flush(); + }); + + it('Returns error from the data API if there is one', function (done) { + $httpBackend + .expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true') + .respond(404); + + const file = new Blob(['foo,bar'], {type : 'text/csv'}); + + ingest.uploadCSV(file, 'foo', ';', true) + .then( + () => { + throw new Error('expected an error response'); + }, + (error) => { + expect(error.status).to.be(404); + done(); + } + ); + + $httpBackend.flush(); + }); + }); + describe('getProcessors', () => { it('Calls the processors GET endpoint of the ingest API', function () { $httpBackend - .expectGET('../api/kibana/ingest/processors') + .expectGET('/api/kibana/ingest/processors') .respond('ok'); ingest.getProcessors(); @@ -127,7 +169,7 @@ describe('Ingest Service', function () { it('Throws user-friendly error when there is an error in the request', function (done) { $httpBackend - .when('GET', '../api/kibana/ingest/processors') + .when('GET', '/api/kibana/ingest/processors') .respond(404); ingest.getProcessors() diff --git a/src/ui/public/ingest/ingest.js b/src/ui/public/ingest/ingest.js index bee1cb564d3af..4d31c8271827a 100644 --- a/src/ui/public/ingest/ingest.js +++ b/src/ui/public/ingest/ingest.js @@ -1,10 +1,13 @@ +import PluginsKibanaSettingsSectionsIndicesRefreshKibanaIndexProvider from 'plugins/kibana/settings/sections/indices/_refresh_kibana_index'; import { keysToCamelCaseShallow, keysToSnakeCaseShallow } from '../../../plugins/kibana/common/lib/case_conversion'; import _ from 'lodash'; import angular from 'angular'; +import chrome from 'ui/chrome'; -export default function IngestProvider($rootScope, $http, config, $q) { +export default function IngestProvider($rootScope, $http, config, $q, Private, indexPatterns) { - const ingestAPIPrefix = '../api/kibana/ingest'; + const ingestAPIPrefix = chrome.addBasePath('/api/kibana/ingest'); + const refreshKibanaIndex = Private(PluginsKibanaSettingsSectionsIndicesRefreshKibanaIndexProvider); this.save = function (indexPattern, pipeline) { if (_.isEmpty(indexPattern)) { @@ -24,6 +27,7 @@ export default function IngestProvider($rootScope, $http, config, $q) { config.set('defaultIndex', indexPattern.id); } + indexPatterns.getIds.clearCache(); $rootScope.$broadcast('ingest:updated'); }); }; @@ -35,6 +39,7 @@ export default function IngestProvider($rootScope, $http, config, $q) { return $http.delete(`${ingestAPIPrefix}/${ingestId}`) .then(() => { + indexPatterns.getIds.clearCache(); $rootScope.$broadcast('ingest:updated'); }); }; @@ -71,4 +76,30 @@ export default function IngestProvider($rootScope, $http, config, $q) { }); }; + this.uploadCSV = function (file, indexPattern, delimiter, pipeline) { + if (_.isUndefined(file)) { + throw new Error('file is required'); + } + if (_.isUndefined(indexPattern)) { + throw new Error('index pattern is required'); + } + + const formData = new FormData(); + formData.append('csv', file); + + const params = {}; + if (!_.isUndefined(delimiter)) { + params.csv_delimiter = delimiter; + } + if (!_.isUndefined(pipeline)) { + params.pipeline = pipeline; + } + + return $http.post(chrome.addBasePath(`/api/kibana/${indexPattern}/_data`), formData, { + params: params, + transformRequest: angular.identity, + headers: {'Content-Type': undefined} + }); + }; + } diff --git a/src/ui/public/styles/variables/for-theme.less b/src/ui/public/styles/variables/for-theme.less index 90a8e5f2ef5d7..5d455990e37ad 100644 --- a/src/ui/public/styles/variables/for-theme.less +++ b/src/ui/public/styles/variables/for-theme.less @@ -138,6 +138,9 @@ @settings-filebeat-wizard-processor-container-overlay-bg: fade(#000, 10%); +// Settings - Add Data Wizard - Parse CSV +@settings-add-data-wizard-parse-csv-container-border: @kibanaBlue3; + // Visualize =================================================================== @visualize-show-spy-border: @gray-lighter; @visualize-show-spy-bg: @white; diff --git a/src/ui/public/url/__tests__/url.js b/src/ui/public/url/__tests__/url.js index 10319796c2443..e6be67b527a6b 100644 --- a/src/ui/public/url/__tests__/url.js +++ b/src/ui/public/url/__tests__/url.js @@ -272,6 +272,27 @@ describe('kbnUrl', function () { expect($location.search()).to.eql({}); expect($location.hash()).to.be(''); }); + + it('should allow setting app state on the target url', function () { + let path = '/test/path'; + let search = {search: 'test'}; + let hash = 'hash'; + let newPath = '/new/location'; + + $location.path(path).search(search).hash(hash); + + // verify the starting state + expect($location.path()).to.be(path); + expect($location.search()).to.eql(search); + expect($location.hash()).to.be(hash); + + kbnUrl.change(newPath, null, {foo: 'bar'}); + + // verify the ending state + expect($location.path()).to.be(newPath); + expect($location.search()).to.eql({_a: '(foo:bar)'}); + expect($location.hash()).to.be(''); + }); }); describe('changePath', function () { @@ -319,6 +340,27 @@ describe('kbnUrl', function () { expect($location.hash()).to.be(''); }); + it('should allow setting app state on the target url', function () { + let path = '/test/path'; + let search = {search: 'test'}; + let hash = 'hash'; + let newPath = '/new/location'; + + $location.path(path).search(search).hash(hash); + + // verify the starting state + expect($location.path()).to.be(path); + expect($location.search()).to.eql(search); + expect($location.hash()).to.be(hash); + + kbnUrl.redirect(newPath, null, {foo: 'bar'}); + + // verify the ending state + expect($location.path()).to.be(newPath); + expect($location.search()).to.eql({_a: '(foo:bar)'}); + expect($location.hash()).to.be(''); + }); + it('should replace the current history entry', function () { sinon.stub($location, 'replace'); $location.url('/some/path'); diff --git a/src/ui/public/url/url.js b/src/ui/public/url/url.js index cd3ed891e28fb..e226166b97cf1 100644 --- a/src/ui/public/url/url.js +++ b/src/ui/public/url/url.js @@ -2,6 +2,7 @@ import _ from 'lodash'; import 'ui/filters/uriescape'; import 'ui/filters/rison'; import uiModules from 'ui/modules'; +import rison from 'rison-node'; uiModules.get('kibana/url') @@ -17,8 +18,8 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA * @param {Object} [paramObj] - optional set of parameters for the url template * @return {undefined} */ - self.change = function (url, paramObj) { - self._changeLocation('url', url, paramObj); + self.change = function (url, paramObj, appState) { + self._changeLocation('url', url, paramObj, false, appState); }; /** @@ -40,8 +41,8 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA * @param {Object} [paramObj] - optional set of parameters for the url template * @return {undefined} */ - self.redirect = function (url, paramObj) { - self._changeLocation('url', url, paramObj, true); + self.redirect = function (url, paramObj, appState) { + self._changeLocation('url', url, paramObj, true, appState); }; /** @@ -142,7 +143,7 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA ///// let reloading; - self._changeLocation = function (type, url, paramObj, replace) { + self._changeLocation = function (type, url, paramObj, replace, appState) { let prev = { path: $location.path(), search: $location.search() @@ -152,6 +153,10 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA $location[type](url); if (replace) $location.replace(); + if (appState) { + $location.search('_a', rison.encode(appState)); + } + let next = { path: $location.path(), search: $location.search() diff --git a/test/unit/api/ingest/_data.js b/test/unit/api/ingest/_data.js index f8fbdd14b6208..4d2a4e5249f18 100644 --- a/test/unit/api/ingest/_data.js +++ b/test/unit/api/ingest/_data.js @@ -103,6 +103,16 @@ define(function (require) { }); }); + bdd.it('should use the filename and line numbers as document IDs', function () { + return request.post('/kibana/names/_data') + .attach('csv', 'test/unit/fixtures/fake_names_with_mapping_errors.csv') + .expect(200) + .then((dataResponse) => { + const id = dataResponse.body[0].errors.index[0]._id; + expect(id).to.be('fake_names_with_mapping_errors.csv:2'); + }); + }); + bdd.it('should report any csv parsing errors under an "errors.other" key', function () { return request.post('/kibana/names/_data') .attach('csv', 'test/unit/fixtures/fake_names_with_parse_errors.csv')