Skip to content

Commit

Permalink
Merge pull request elastic#7372 from elastic/feature/ingest
Browse files Browse the repository at this point in the history
Add Data - CSV

Former-commit-id: 15a4fa1
  • Loading branch information
Matt Bargar authored Jun 17, 2016
2 parents 6c6c0b7 + 19f5b76 commit 19de1fe
Show file tree
Hide file tree
Showing 213 changed files with 9,708 additions and 180 deletions.
5 changes: 5 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
},
"dependencies": {
"@bigfunger/decompress-zip": "0.2.0-stripfix2",
"@bigfunger/jsondiffpatch": "0.1.38-webpack",
"@elastic/datemath": "2.3.0",
"@spalger/angular-bootstrap": "0.12.1",
"@spalger/filesaver": "1.1.2",
Expand All @@ -76,6 +77,7 @@
"@spalger/numeral": "^2.0.0",
"@spalger/test-subj-selector": "0.2.1",
"@spalger/ui-ace": "0.2.3",
"JSONStream": "1.1.1",
"angular": "1.4.7",
"angular-bootstrap-colorpicker": "3.0.19",
"angular-elastic": "2.5.0",
Expand All @@ -95,6 +97,7 @@
"clipboard": "1.5.5",
"commander": "2.8.1",
"css-loader": "0.17.0",
"csv-parse": "1.1.0",
"d3": "3.5.6",
"dragula": "3.7.0",
"elasticsearch": "10.1.2",
Expand All @@ -110,6 +113,7 @@
"good-squeeze": "2.1.0",
"gridster": "0.5.6",
"hapi": "8.8.1",
"highland": "2.7.2",
"httpolyglot": "0.1.1",
"imports-loader": "0.6.4",
"jade": "1.11.0",
Expand All @@ -130,6 +134,7 @@
"moment": "2.13.0",
"moment-timezone": "0.5.4",
"node-uuid": "1.4.7",
"papaparse": "4.1.2",
"raw-loader": "0.5.1",
"request": "2.61.0",
"rimraf": "2.4.3",
Expand Down
3 changes: 3 additions & 0 deletions src/cli/cluster/base_path_proxy.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ export default class BasePathProxy {
config.set('server.basePath', this.basePath);
}

const ONE_GIGABYTE = 1024 * 1024 * 1024;
config.set('server.maxPayloadBytes', ONE_GIGABYTE);

setupLogging(null, this.server, config);
setupConnection(null, this.server, config);
this.setupRoutes();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import expect from 'expect.js';
import {patternToIngest, ingestToPattern} from '../convert_pattern_and_ingest_name';

describe('convertPatternAndTemplateName', function () {

describe('ingestToPattern', function () {

it('should convert an index template\'s name to its matching index pattern\'s title', function () {
expect(ingestToPattern('kibana-logstash-*')).to.be('logstash-*');
});

it('should throw an error if the template name isn\'t a valid kibana namespaced name', function () {
expect(ingestToPattern).withArgs('logstash-*').to.throwException('not a valid kibana namespaced template name');
expect(ingestToPattern).withArgs('').to.throwException(/not a valid kibana namespaced template name/);
});

});

describe('patternToIngest', function () {

it('should convert an index pattern\'s title to its matching index template\'s name', function () {
expect(patternToIngest('logstash-*')).to.be('kibana-logstash-*');
});

it('should throw an error if the pattern is empty', function () {
expect(patternToIngest).withArgs('').to.throwException(/pattern must not be empty/);
});

});

});
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@
// This module provides utility functions for easily converting between template and pattern names.

module.exports = {
templateToPattern: (templateName) => {
ingestToPattern: (templateName) => {
if (templateName.indexOf('kibana-') === -1) {
throw new Error('not a valid kibana namespaced template name');
}

return templateName.slice(templateName.indexOf('-') + 1);
},

patternToTemplate: (patternName) => {
patternToIngest: (patternName) => {
if (patternName === '') {
throw new Error('pattern must not be empty');
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ <h4>Time-interval based index patterns are deprecated!</h4>
ng-attr-placeholder="{{index.defaultName}}"
ng-model-options="{ updateOn: 'default blur', debounce: {'default': 2500, 'blur': 0} }"
validate-index-name
allow-wildcard
name="name"
required
type="text"
Expand Down Expand Up @@ -167,6 +168,7 @@ <h4>Time-interval based index patterns are deprecated!</h4>
</div>

<button
data-test-subj="submitCreateIndexPatternFromExistingForm"
ng-disabled="form.$invalid || index.fetchFieldsError"
ng-class="index.fetchFieldsError ? 'btn-default' : 'btn-success'"
type="submit"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<kbn-management-app section="kibana">
<kbn-management-indices>
<div ng-controller="managementIndicesEdit">
<div ng-controller="managementIndicesEdit" data-test-subj="editIndexPattern">
<div class="page-header">
<kbn-management-index-header
index-pattern="indexPattern"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import IndicesFieldTypesProvider from 'plugins/kibana/management/sections/indice
import uiRoutes from 'ui/routes';
import uiModules from 'ui/modules';
import editTemplate from 'plugins/kibana/management/sections/indices/_edit.html';
import IngestProvider from 'ui/ingest';

uiRoutes
.when('/management/kibana/indices/:indexPatternId?', {
Expand All @@ -32,6 +33,7 @@ uiModules.get('apps/management')
const notify = new Notifier();
const $state = $scope.state = new AppState();
const refreshKibanaIndex = Private(RefreshKibanaIndex);
const ingest = Private(IngestProvider);

$scope.kbnUrl = Private(UrlProvider);
$scope.indexPattern = $route.current.locals.indexPattern;
Expand Down Expand Up @@ -68,8 +70,8 @@ uiModules.get('apps/management')
}
}

courier.indexPatterns.delete($scope.indexPattern)
.then(refreshKibanaIndex)
ingest.delete($scope.indexPattern.id)
.then($scope.indexPattern.destroy.bind($scope.indexPattern))
.then(function () {
$location.url('/management/data/index');
})
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
<h2><em>Follow these instructions to install Filebeat.</em>
Now that you've got a fresh pipeline and index pattern, let's throw some data at it!
</h2>

<div class="install-filebeat">
<ol>
<li>
<span>
<strong>Install Filebeat</strong> on all servers on which you want to tail logs &nbsp;
<a target="_blank" ng-href="{{installStep.docLinks.installation}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>
</li>

<li>
<span>
<strong>Point Filebeat</strong> at the log files you want to tail &nbsp;
<a target="_blank" ng-href="{{installStep.docLinks.configuration}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>
</li>

<li ng-if="installStep.results.pipeline.processors.length">
<span>
<strong>Configure Filebeat</strong> to send data through your new Elasticsearch pipeline &nbsp;
<a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a><br/>
At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname, an index name, and a
<a target="_blank"
ng-href="{{installStep.docLinks.elasticsearchOutputAnchorParameters}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> paramaters
</a> block. Your config should end up looking something like this:<br/>
<pre>
output:
elasticsearch:
hosts: ["your-elasticsearch-host"]
index: "your-base-index-name"
parameters:
pipeline: "{{installStep.pipelineId}}"</pre>
<em>NOTE</em>: The Filebeat config takes a base index name and automatically rotates the target index by appending "-{date}"
to the end, so if your pattern was "filebeat-*" you would make the index name "filebeat" in filebeat.yml.<br />
</span>
</li>


<li ng-if="!installStep.results.pipeline.processors.length">
<span>
<strong>Configure Filebeat</strong> to send data to Elasticsearch &nbsp;
<a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a><br/>
At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname and an index name.
Your config should end up looking something like this:<br />
<pre>
output:
elasticsearch:
hosts: ["your-elasticsearch-host"]
index: "your-base-index-name"</pre>
<em>NOTE</em>: The Filebeat config takes a base index name and automatically rotates the target index by appending "-{date}"
to the end, so if your pattern was "filebeat-*" you would make the index name "filebeat" in filebeat.yml.<br />
</span>
</li>

<li>
<span>
<strong>Run Filebeat</strong> on each server &nbsp;
<a target="_blank" ng-href="{{installStep.docLinks.startup}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>
</li>
<li>
<span>
<strong>Verify your filebeat installation below.</strong> We'll poll your new index pattern for documents and let you know when
they show up. If you'd like to skip this step, simply click Done now.
</span>
</li>
</ol>
</div>

<pattern-checker pattern="installStep.results.indexPattern.id"/>
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import modules from 'ui/modules';
import template from './install_filebeat_step.html';
import 'ui/pattern_checker';
import { patternToIngest } from '../../../../../../common/lib/convert_pattern_and_ingest_name';
import { filebeat as docLinks } from '../../../../../../../../ui/public/documentation_links/documentation_links';
import './styles/_add_data_install_filebeat_step.less';

modules.get('apps/management')
.directive('installFilebeatStep', function () {
return {
template: template,
scope: {
results: '='
},
bindToController: true,
controllerAs: 'installStep',
controller: function ($scope) {
this.pipelineId = patternToIngest(this.results.indexPattern.id);
this.docLinks = docLinks;
}
};
});

Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
install-filebeat-step {

.install-filebeat {
> ol {
padding-left: 1em;

> li {
padding: 4px 0;
font-weight: bold;

> span {
font-weight: normal;

> pre {
margin: 7px 0;
}
}
}
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<file-upload ng-if="!wizard.file" on-locate="wizard.file = file" upload-selector="button.upload">
<h2><em>Pick a CSV file to get started.</em>
Please follow the instructions below.
</h2>

<div class="upload-wizard-file-upload-container">
<div class="upload-instructions">Drop your file here</div>
<div class="upload-instructions-separator">or</div>
<button class="btn btn-primary btn-lg controls upload" ng-click>
Select File
</button>
<div>Maximum upload file size: 1 GB</div>
</div>
</file-upload>

<div class="upload-wizard-file-preview-container" ng-if="wizard.file">
<h2><em>Review the sample below.</em>
Click next if it looks like we parsed your file correctly.
</h2>

<div ng-if="!!wizard.formattedErrors.length" class="alert alert-danger parse-error">
<ul>
<li ng-repeat="error in wizard.formattedErrors track by $index">{{ error }}</li>
</ul>
</div>

<div ng-if="!!wizard.formattedWarnings.length" class="alert alert-warning">
<ul>
<li ng-repeat="warning in wizard.formattedWarnings track by $index">{{ warning }}</li>
</ul>
</div>

<div class="advanced-options form-inline">
<span class="form-group">
<label>Delimiter</label>
<select ng-model="wizard.parseOptions.delimiter"
ng-options="option.value as option.label for option in wizard.delimiterOptions"
class="form-control">
</select>
</span>
<span class="form-group">
<label>Filename:</label>
{{ wizard.file.name }}
</span>
</div>

<div class="preview">
<table class="table table-condensed">
<thead>
<tr>
<th ng-repeat="col in wizard.columns track by $index">
<span title="{{ col }}">{{ col | limitTo:12 }}{{ col.length > 12 ? '...' : '' }}</span>
</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="row in wizard.rows">
<td ng-repeat="cell in row track by $index">{{ cell }}</td>
</tr>
</tbody>
</table>
</div>
</div>
Loading

0 comments on commit 19de1fe

Please sign in to comment.