diff --git a/filebeat/Makefile b/filebeat/Makefile index eab8ca54ddc..202f1b71f10 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -12,6 +12,14 @@ include ../libbeat/scripts/Makefile .PHONY: before-build before-build: -# Collects all dependencies and then calls update +# Collects all module and dataset fields +.PHONY: fields +fields: + mkdir -p _meta/ + cat ${ES_BEATS}/filebeat/_meta/fields.common.yml > _meta/fields.generated.yml + . ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml + + +# Runs all collection steps and updates afterwards .PHONY: collect -collect: +collect: fields update diff --git a/filebeat/_meta/fields.yml b/filebeat/_meta/fields.common.yml similarity index 100% rename from filebeat/_meta/fields.yml rename to filebeat/_meta/fields.common.yml diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-Nginx-Dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-Nginx-Dashboard.json new file mode 100644 index 00000000000..ef994978e4c --- /dev/null +++ b/filebeat/_meta/kibana/dashboard/Filebeat-Nginx-Dashboard.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "Filebeat Nginx Dashboard", + "uiStateJSON": "{\"P-4\":{\"vis\":{\"legendOpen\":true}},\"P-8\":{\"mapCenter\":[50.51342652633956,-0.17578125]}}", + "panelsJSON": "[{\"col\":9,\"id\":\"Errors-over-time\",\"panelIndex\":2,\"row\":4,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Nginx-Access-Browsers\",\"panelIndex\":3,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":5,\"id\":\"Nginx-Access-OSes\",\"panelIndex\":4,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"New-Visualization\",\"panelIndex\":5,\"row\":4,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Nginx-Access-Response-codes-by-top-URLs\",\"panelIndex\":6,\"row\":7,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Sent-sizes\",\"panelIndex\":7,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"Nginx-Access-Map\",\"type\":\"visualization\",\"panelIndex\":8,\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/search/Filebeat-Nginx-module.json b/filebeat/_meta/kibana/search/Filebeat-Nginx-module.json new file mode 100644 index 00000000000..5b1e4518b4f --- /dev/null +++ b/filebeat/_meta/kibana/search/Filebeat-Nginx-module.json @@ -0,0 +1,16 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Filebeat Nginx module", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "_source" + ] +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Errors-over-time.json b/filebeat/_meta/kibana/visualization/Errors-over-time.json new file mode 100644 index 00000000000..bd100926a1b --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Errors-over-time.json @@ -0,0 +1,10 @@ +{ + "visState": "{\n \"title\": \"Errors over time\",\n \"type\": \"area\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"smoothLines\": false,\n \"scale\": \"linear\",\n \"interpolate\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"nginx.error.level\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "title": "Nginx Errors over time", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/New-Visualization.json b/filebeat/_meta/kibana/visualization/New-Visualization.json new file mode 100644 index 00000000000..55499a624b6 --- /dev/null +++ b/filebeat/_meta/kibana/visualization/New-Visualization.json @@ -0,0 +1,11 @@ +{ + "visState": "{\n \"title\": \"New Visualization\",\n \"type\": \"histogram\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"scale\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"nginx.access.response_code\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "title": "Nginx Access over time", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "savedSearchId": "Filebeat-Nginx-module", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": []\n}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Nginx-Access-Browsers.json b/filebeat/_meta/kibana/visualization/Nginx-Access-Browsers.json new file mode 100644 index 00000000000..85e76ddd2de --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Nginx-Access-Browsers.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Nginx Access Browsers\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"nginx.access.user_agent.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"nginx.access.user_agent.major\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Nginx Access Browsers", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Nginx-Access-Map.json b/filebeat/_meta/kibana/visualization/Nginx-Access-Map.json new file mode 100644 index 00000000000..c4799c20806 --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Nginx-Access-Map.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"nginx.access.geoip.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"Nginx Access Map\",\"type\":\"tile_map\"}", + "description": "", + "title": "Nginx Access Map", + "uiStateJSON": "{\"mapCenter\":[12.039320557540572,-0.17578125]}", + "version": 1, + "savedSearchId": "Filebeat-Nginx-module", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Nginx-Access-OSes.json b/filebeat/_meta/kibana/visualization/Nginx-Access-OSes.json new file mode 100644 index 00000000000..e13503299e3 --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Nginx-Access-OSes.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Nginx Access OSes\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"nginx.access.user_agent.os_name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"nginx.access.user_agent.os_major\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Nginx Access OSes", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Nginx-Access-Response-codes-by-top-URLs.json b/filebeat/_meta/kibana/visualization/Nginx-Access-Response-codes-by-top-URLs.json new file mode 100644 index 00000000000..a4075c3ee7d --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Nginx-Access-Response-codes-by-top-URLs.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Nginx Access Response codes by top URLs\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"nginx.access.url\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":false}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"nginx.access.response_code\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Nginx Access Response codes by top URLs", + "uiStateJSON": "{\"vis\":{\"colors\":{\"200\":\"#629E51\",\"404\":\"#0A50A1\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/Sent-sizes.json b/filebeat/_meta/kibana/visualization/Sent-sizes.json new file mode 100644 index 00000000000..f18b75419f3 --- /dev/null +++ b/filebeat/_meta/kibana/visualization/Sent-sizes.json @@ -0,0 +1,10 @@ +{ + "visState": "{\n \"title\": \"Sent sizes\",\n \"type\": \"line\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"showCircles\": true,\n \"smoothLines\": true,\n \"interpolate\": \"linear\",\n \"scale\": \"linear\",\n \"drawLinesBetweenPoints\": true,\n \"radiusRatio\": \"17\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"nginx.access.body_sent.bytes\",\n \"customLabel\": \"Data sent\"\n }\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"radius\",\n \"params\": {}\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "title": "Nginx Sent Byte Size", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": [],\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"_exists_:nginx.access\",\n \"analyze_wildcard\": true\n }\n },\n \"highlight\": {\n \"pre_tags\": [\n \"@kibana-highlighted-field@\"\n ],\n \"post_tags\": [\n \"@/kibana-highlighted-field@\"\n ],\n \"fields\": {\n \"*\": {}\n },\n \"require_field_match\": false,\n \"fragment_size\": 2147483647\n }\n}" + } +} \ No newline at end of file diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index fba2e27078e..f807f668dfd 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> -- [[exported-fields-beat]] @@ -174,3 +175,259 @@ required: True The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. +[[exported-fields-nginx]] +== Nginx Fields + +Module for parsing the Nginx log files. + + + +[float] +== nginx Fields + +Fields from the Nginx log files. + + + +[float] +== access Fields + +Contains fields for the Nginx access logs. + + + +[float] +=== nginx.access.remote_ip + +type: keyword + +Client IP address. + + +[float] +=== nginx.access.user_name + +type: keyword + +The user name used when basic authentication is used. + + +[float] +=== nginx.access.method + +type: keyword + +example: GET + +The request HTTP method. + + +[float] +=== nginx.access.url + +type: keyword + +The request HTTP URL. + + +[float] +=== nginx.access.http_version + +type: keyword + +The HTTP version. + + +[float] +=== nginx.access.response_code + +type: long + +The HTTP response code. + + +[float] +=== nginx.access.body_sent.bytes + +type: long + +format: bytes + +The number of bytes of the server response body. + + +[float] +=== nginx.access.referrer + +type: keyword + +The HTTP referrer. + + +[float] +=== nginx.access.agent + +type: text + +Contains the un-parsed user agent string. Only present if the user agent Elasticsearch plugin is not available or not used. + + +[float] +== user_agent Fields + +Contains the parsed User agent field. Only present if the user agent Elasticsearch plugin is available and used. + + + +[float] +=== nginx.access.user_agent.device + +type: keyword + +The name of the physical device. + + +[float] +=== nginx.access.user_agent.major + +type: long + +The major version of the user agent. + + +[float] +=== nginx.access.user_agent.minor + +type: long + +The minor version of the user agent. + + +[float] +=== nginx.access.user_agent.patch + +type: long + +The patch version of the user agent. + + +[float] +=== nginx.access.user_agent.name + +type: keyword + +example: Chrome + +The name of the user agent. + + +[float] +=== nginx.access.user_agent.os + +type: keyword + +The name of the operating system. + + +[float] +=== nginx.access.user_agent.os_major + +type: long + +The major version of the operating system. + + +[float] +=== nginx.access.user_agent.os_minor + +type: long + +The minor version of the operating system. + + +[float] +=== nginx.access.user_agent.os_name + +type: keyword + +The name of the operating system. + + +[float] +== geoip Fields + +Contains GeoIP information gathered based on the remote_ip field. Only present if the GeoIP Elasticsearch plugin is available and used. + + + +[float] +=== nginx.access.geoip.continent_name + +type: keyword + +The name of the continent. + + +[float] +=== nginx.access.geoip.country_iso_code + +type: keyword + +Country ISO code. + + +[float] +=== nginx.access.geoip.location + +type: geo_point + +The longitude and latitude. + + +[float] +== error Fields + +Contains fields for the Nginx error logs. + + + +[float] +=== nginx.error.level + +type: keyword + +Error level (e.g. error, critical). + + +[float] +=== nginx.error.pid + +type: long + +Process identifier (PID). + + +[float] +=== nginx.error.tid + +type: long + +Thread identifier. + + +[float] +=== nginx.error.connection_id + +type: long + +Connection identifier. + + +[float] +=== nginx.error.message + +type: text + +The error message + + diff --git a/filebeat/filebeat.py b/filebeat/filebeat.py new file mode 100755 index 00000000000..4bd7aef9c7c --- /dev/null +++ b/filebeat/filebeat.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +import argparse +import sys +import os +import yaml +import requests +import tempfile +import subprocess +from jinja2 import Template + + +def main(): + parser = argparse.ArgumentParser( + description="PROTOTYPE: start filebeat with a module configuration") + parser.add_argument("--module", default="", + help="From branch") + parser.add_argument("--nginx", action="store_true", + help="Shortcut for --module nginx") + parser.add_argument("--es", default="http://localhost:9200", + help="Elasticsearch URL") + parser.add_argument("-E", nargs="*", type=str, default=None, + help="Variables overrides. e.g. path=/test") + + args = parser.parse_args() + print args + + if args.nginx: + args.module = "nginx" + + if args.module == "": + print("You need to specify a module") + sys.exit(1) + + load_dashboards(args) + load_datasets(args, args.module) + + +def load_dashboards(args): + cmd = ["../libbeat/dashboards/import_dashboards", + "-dir", "etc/kibana", + "-es", args.es] + subprocess.Popen(cmd).wait() + + +def load_datasets(args, module): + path = os.path.join("module", module) + if not os.path.isdir(path): + print("Module {} not found".format(module)) + sys.exit(1) + print("Found module {} in {}".format(module, path)) + + filesets = [name for name in os.listdir(path) if + os.path.isfile(os.path.join(path, name, "manifest.yml"))] + + print("Found filesets: {}".format(filesets)) + + prospectors = "" + for fileset in filesets: + prospectors += load_fileset(args, module, fileset, + os.path.join(path, fileset)) + + run_filebeat(args, prospectors) + + print("Generated configuration: {}".format(prospectors)) + + +def load_fileset(args, module, fileset, path): + manifest = yaml.load(file(os.path.join(path, "manifest.yml"), "r")) + var = evaluate_vars(args, manifest["vars"]) + var["beat"] = dict(module=module, fileset=fileset, path=path, args=args) + print("Evaluated variables: {}".format(var)) + + load_pipeline(var, manifest["ingest_pipeline"]) + generate_prospectors(var, manifest["prospectors"]) + + return var["beat"]["prospectors"] + + +def evaluate_vars(args, var_in): + var = {} + for name, vals in var_in.items(): + var[name] = vals["default"] + + if sys.platform == "darwin" and "os.darwin" in vals: + var[name] = vals["os.darwin"] + elif sys.platform == "windows" and "os.windows" in vals: + var[name] = vals["os.windows"] + + # overrides + if args.E is not None: + for pair in args.E: + key, val = pair.partition("=")[::2] + var[key] = val + + return var + + +def load_pipeline(var, pipeline): + path = os.path.join(var["beat"]["path"], Template(pipeline).render(var)) + print("Loading ingest pipeline: {}".format(path)) + var["beat"]["pipeline_id"] = var["beat"]["module"] + '-' + var["beat"]["fileset"] + \ + '-' + os.path.splitext(os.path.basename(path))[0] + print("Pipeline id: {}".format(var["beat"]["pipeline_id"])) + + with open(path, "r") as f: + contents = f.read() + + r = requests.put("{}/_ingest/pipeline/{}" + .format(var["beat"]["args"].es, + var["beat"]["pipeline_id"]), + data=contents) + if r.status_code >= 300: + print("Error posting template: {}".format(r.text)) + + +def run_filebeat(args, prospectors): + cfg_template = """ +filebeat.prospectors: +{{prospectors}} + +output.elasticsearch.hosts: ["{{es}}"] +output.elasticsearch.pipeline: "%{[fields.pipeline_id]}" +""" + fd, fname = tempfile.mkstemp(suffix=".yml", prefix="filebeat-", + text=True) + with open(fname, "w") as cfgfile: + cfgfile.write(Template(cfg_template).render( + dict(prospectors=prospectors, es=args.es))) + print("Wrote configuration file: {}".format(cfgfile.name)) + os.close(fd) + + cmd = ["./filebeat", "-e", "-c", cfgfile.name, "-d", "*"] + + subprocess.Popen(cmd).wait() + + +def generate_prospectors(var, prospectors): + var["beat"]["prospectors"] = "" + for pr in prospectors: + path = os.path.join(var["beat"]["path"], Template(pr).render(var)) + with open(path, "r") as f: + contents = Template(f.read()).render(var) + var["beat"]["prospectors"] += "\n" + contents + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/filebeat/filebeat.template-es2x.json b/filebeat/filebeat.template-es2x.json index 4330d6c2cc4..294cf6d5388 100644 --- a/filebeat/filebeat.template-es2x.json +++ b/filebeat/filebeat.template-es2x.json @@ -94,6 +94,142 @@ } } }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "index": "analyzed", + "norms": { + "enabled": false + }, + "type": "string" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "country_iso_code": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "location": { + "type": "geo_point" + } + } + }, + "http_version": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "method": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "referrer": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "remote_ip": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "os": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "patch": { + "type": "long" + } + } + }, + "user_name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "message": { + "index": "analyzed", + "norms": { + "enabled": false + }, + "type": "string" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, "offset": { "type": "long" }, diff --git a/filebeat/filebeat.template.json b/filebeat/filebeat.template.json index 519b7881980..d1e8858e8a3 100644 --- a/filebeat/filebeat.template.json +++ b/filebeat/filebeat.template.json @@ -78,6 +78,123 @@ } } }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "norms": false, + "type": "text" + }, + "body_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "geoip": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "http_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_code": { + "type": "long" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "os_major": { + "type": "long" + }, + "os_minor": { + "type": "long" + }, + "os_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "patch": { + "type": "long" + } + } + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "pid": { + "type": "long" + }, + "tid": { + "type": "long" + } + } + } + } + }, "offset": { "type": "long" }, diff --git a/filebeat/module/nginx/_meta/fields.yml b/filebeat/module/nginx/_meta/fields.yml new file mode 100644 index 00000000000..8f9cef37499 --- /dev/null +++ b/filebeat/module/nginx/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: nginx + title: "Nginx" + description: > + Module for parsing the Nginx log files. + fields: + - name: nginx + type: group + description: > + Fields from the Nginx log files. + fields: diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml new file mode 100644 index 00000000000..ca8e9282461 --- /dev/null +++ b/filebeat/module/nginx/access/_meta/fields.yml @@ -0,0 +1,107 @@ +- name: access + type: group + description: > + Contains fields for the Nginx access logs. + fields: + - name: remote_ip + type: keyword + description: > + Client IP address. + - name: user_name + type: keyword + description: > + The user name used when basic authentication is used. + - name: method + type: keyword + example: GET + description: > + The request HTTP method. + - name: url + type: keyword + description: > + The request HTTP URL. + - name: http_version + type: keyword + description: > + The HTTP version. + - name: response_code + type: long + description: > + The HTTP response code. + - name: body_sent.bytes + type: long + format: bytes + description: > + The number of bytes of the server response body. + - name: referrer + type: keyword + description: > + The HTTP referrer. + - name: agent + type: text + description: > + Contains the un-parsed user agent string. Only present if the user + agent Elasticsearch plugin is not available or not used. + - name: user_agent + type: group + description: > + Contains the parsed User agent field. Only present if the user + agent Elasticsearch plugin is available and used. + fields: + - name: device + type: keyword + description: > + The name of the physical device. + - name: major + type: long + description: > + The major version of the user agent. + - name: minor + type: long + description: > + The minor version of the user agent. + - name: patch + type: long + description: > + The patch version of the user agent. + - name: name + type: keyword + example: Chrome + description: > + The name of the user agent. + - name: os + type: keyword + description: > + The name of the operating system. + - name: os_major + type: long + description: > + The major version of the operating system. + - name: os_minor + type: long + description: > + The minor version of the operating system. + - name: os_name + type: keyword + description: > + The name of the operating system. + - name: geoip + type: group + description: > + Contains GeoIP information gathered based on the remote_ip field. + Only present if the GeoIP Elasticsearch plugin is available and + used. + fields: + - name: continent_name + type: keyword + description: > + The name of the continent. + - name: country_iso_code + type: keyword + description: > + Country ISO code. + - name: location + type: geo_point + description: > + The longitude and latitude. + diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml new file mode 100644 index 00000000000..514a4e37689 --- /dev/null +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -0,0 +1,6 @@ +- input_type: log + paths: + - {{path}}/access*.log + fields: + source_type: nginx-access + pipeline_id: {{beat.pipeline_id}} diff --git a/filebeat/module/nginx/access/ingest/json_with_plugins.json b/filebeat/module/nginx/access/ingest/json_with_plugins.json new file mode 100644 index 00000000000..fb80aca3765 --- /dev/null +++ b/filebeat/module/nginx/access/ingest/json_with_plugins.json @@ -0,0 +1,16 @@ +{ + "description": "pipeline for parsing Nginx logs that are JSON format", + "processors": [{ + "user_agent": { + "field": "agent" + } + }, { + "remove": { + "field": "agent" + } + }, { + "geoip": { + "field": "remote_ip" + } + }] +} diff --git a/filebeat/module/nginx/access/ingest/no_plugins.json b/filebeat/module/nginx/access/ingest/no_plugins.json new file mode 100644 index 00000000000..59d394d4364 --- /dev/null +++ b/filebeat/module/nginx/access/ingest/no_plugins.json @@ -0,0 +1,31 @@ +{ + "description": "Pipeline for parsing Nginx logs. Requires no plugins", + "processors": [{ + "grok": { + "field": "message", + "patterns":[ + "%{IPORHOST:nginx.access.remote_ip} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" + ], + "ignore_missing": true + } + },{ + "remove":{ + "field": "message" + } + }, { + "rename": { + "field": "@timestamp", + "target_field": "beat.read_timestamp" + } + }, { + "date": { + "field": "nginx.access.time", + "target_field": "@timestamp", + "formats": ["dd/MMM/YYYY:H:m:s Z"] + } + }, { + "remove": { + "field": "nginx.access.time" + } + }] +} diff --git a/filebeat/module/nginx/access/ingest/with_plugins.json b/filebeat/module/nginx/access/ingest/with_plugins.json new file mode 100644 index 00000000000..32ccda3329f --- /dev/null +++ b/filebeat/module/nginx/access/ingest/with_plugins.json @@ -0,0 +1,45 @@ +{ + "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", + "processors": [{ + "grok": { + "field": "message", + "patterns":[ + "%{IPORHOST:nginx.access.remote_ip} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" + ], + "ignore_missing": true + } + },{ + "remove":{ + "field": "message" + } + }, { + "rename": { + "field": "@timestamp", + "target_field": "beat.read_timestamp" + } + }, { + "date": { + "field": "nginx.access.time", + "target_field": "@timestamp", + "formats": ["dd/MMM/YYYY:H:m:s Z"] + } + }, { + "remove": { + "field": "nginx.access.time" + } + }, { + "user_agent": { + "field": "nginx.access.agent", + "target_field": "nginx.access.user_agent" + } + }, { + "remove": { + "field": "nginx.access.agent" + } + }, { + "geoip": { + "field": "nginx.access.remote_ip", + "target_field": "nginx.access.geoip" + } + }] +} diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml new file mode 100644 index 00000000000..9cceaa1f89a --- /dev/null +++ b/filebeat/module/nginx/access/manifest.yml @@ -0,0 +1,14 @@ +module_version: 1.0 + +vars: + path: + default: /var/log/nginx + os.darwin: /usr/local/var/log/nginx + os.windows: c:/programfiles/nginx/logs + pipeline: + # options: with_plugins, no_plugins, json_with_plugins, json_no_plugins + default: with_plugins + +ingest_pipeline: ingest/{{pipeline}}.json +prospectors: + - config/nginx-access.yml diff --git a/filebeat/module/nginx/access/test/access.log b/filebeat/module/nginx/access/test/access.log new file mode 100644 index 00000000000..58991d26b3b --- /dev/null +++ b/filebeat/module/nginx/access/test/access.log @@ -0,0 +1,12 @@ +77.179.66.156 - - [25/Oct/2016:14:49:33 +0200] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +77.179.66.156 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +77.179.66.156 - - [25/Oct/2016:14:50:44 +0200] "GET /adsasd HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +77.179.66.156 - - [07/Dec/2016:10:43:18 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +77.179.66.156 - - [07/Dec/2016:10:43:21 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +77.179.66.156 - - [07/Dec/2016:10:43:23 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" diff --git a/filebeat/module/nginx/error/_meta/fields.yml b/filebeat/module/nginx/error/_meta/fields.yml new file mode 100644 index 00000000000..bc908c10a08 --- /dev/null +++ b/filebeat/module/nginx/error/_meta/fields.yml @@ -0,0 +1,25 @@ +- name: error + type: group + description: > + Contains fields for the Nginx error logs. + fields: + - name: level + type: keyword + description: > + Error level (e.g. error, critical). + - name: pid + type: long + description: > + Process identifier (PID). + - name: tid + type: long + description: > + Thread identifier. + - name: connection_id + type: long + description: > + Connection identifier. + - name: message + type: text + description: > + The error message diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml new file mode 100644 index 00000000000..8668545be7a --- /dev/null +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -0,0 +1,7 @@ +- input_type: log + paths: + - {{path}}/error.log + fields: + source_type: nginx-error + pipeline_id: {{beat.pipeline_id}} + diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json new file mode 100644 index 00000000000..dc5a7e4138c --- /dev/null +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -0,0 +1,30 @@ +{ + "description": "Pipeline for parsing the Nginx error logs", + "processors": [{ + "grok": { + "field": "message", + "patterns": [ + "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}" + ] + } + },{ + "remove":{ + "field": "message" + } + }, { + "rename": { + "field": "@timestamp", + "target_field": "beat.read_timestamp" + } + }, { + "date": { + "field": "nginx.error.time", + "target_field": "@timestamp", + "formats": ["YYYY/MM/dd H:m:s"] + } + }, { + "remove": { + "field": "nginx.error.time" + } + }] +} diff --git a/filebeat/module/nginx/error/manifest.yml b/filebeat/module/nginx/error/manifest.yml new file mode 100644 index 00000000000..86c975a7af9 --- /dev/null +++ b/filebeat/module/nginx/error/manifest.yml @@ -0,0 +1,11 @@ +module_version: 1.0 + +vars: + path: + default: /var/log/nginx + os.darwin: /usr/local/var/log/nginx + os.windows: c:/programfiles/nginx/logs + +ingest_pipeline: ingest/pipeline.json +prospectors: + - config/nginx-error.yml diff --git a/filebeat/module/nginx/error/test/error.log b/filebeat/module/nginx/error/test/error.log new file mode 100644 index 00000000000..864d98d17be --- /dev/null +++ b/filebeat/module/nginx/error/test/error.log @@ -0,0 +1,2 @@ +2016/10/25 14:49:34 [error] 54053#0: *1 open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/" +2016/10/25 14:50:44 [error] 54053#0: *3 open() "/usr/local/Cellar/nginx/1.10.2_1/html/adsasd" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /adsasd HTTP/1.1", host: "localhost:8080"