From dd692849911ecc7afd29614454758909cdce512c Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 20 May 2016 14:17:34 +0200 Subject: [PATCH] Updated configuration templates used in tests The goal was to bring them closer to the new conventions we use in the configuration files, so what we test is closer to what we ship in terms of configuration. I also made steps in unifying the libbeat portion of these templates so we could remove the duplication in the future. Part of #1417. --- filebeat/tests/system/config/filebeat.yml.j2 | 225 ++++++------- .../system/config/filebeat_prospectors.yml.j2 | 33 +- filebeat/tests/system/test_fields.py | 2 +- .../tests/system/config/metricbeat.yml.j2 | 201 +++++++----- .../tests/system/config/packetbeat.yml.j2 | 299 ++++++++---------- topbeat/tests/system/config/topbeat.yml.j2 | 149 ++++----- topbeat/tests/system/test_filesystem.py | 4 +- .../tests/system/config/winlogbeat.yml.j2 | 122 +++---- 8 files changed, 511 insertions(+), 524 deletions(-) diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index a3dd55880da5..9dc76fb36b88 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -1,139 +1,111 @@ -############################# Filebeat ###################################### -filebeat: - prospectors: - {% if prospectors is not defined %} - {% set prospectors = true %} - {% endif %} - {% if prospectors %} - - input_type: {{input_type | default("log") }} - # Paths that should be crawled and fetched - {% if path %}paths: - - {{ path }}{% endif %} - # Type of the files. Annotated in every documented - scan_frequency: {{scan_frequency | default("0.1s") }} - ignore_older: {{ignoreOlder}} - close_older: {{closeOlder}} - harvester_buffer_size: - encoding: {{encoding | default("utf-8") }} - tail_files: {{tailFiles}} - backoff: 0.1s - backoff_factor: 1 - max_backoff: 0.1s - force_close_files: {{force_close_files}} - - {% if fields %} - fields: - {% for k,v in fields.items() %} - {{k}}: {{v}} - {% endfor %} - {% endif %} - - fields_under_root: {{"true" if fieldsUnderRoot else "false"}} - - {% if include_lines %} - include_lines: {{include_lines}} - {% endif %} - - {% if exclude_lines %} - exclude_lines: {{exclude_lines}} - {% endif %} - - {% if exclude_files %} - exclude_files: {{exclude_files}} - {% endif %} - - max_bytes: {{ max_bytes|default(10485760) }} - - {% if json %} - json: - {% if json.message_key %}message_key: {{json.message_key}}{% endif %} - {% if json.keys_under_root %}keys_under_root: true{% endif %} - {% if json.overwrite_keys %}overwrite_keys: true{% endif %} - {% if json.add_error_key %}add_error_key: true{% endif %} - {% endif %} - - {% if multiline %} - multiline: - pattern: {{pattern}} - negate: {{negate}} - match: {{match}} - timeout: 1s - max_lines: {{ max_lines|default(500) }} - {% endif %} - {% endif %} - spool_size: - idle_timeout: 0.1s - {% if not skip_registry_config %} - registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} - {%endif%} - - -############################# Shipper ############################################ +###################### Filebeat Config Template ############################### + +filebeat.prospectors: +{% if prospectors is not defined %} +{% set prospectors = true %} +{% endif %} +{% if prospectors %} +- input_type: {{input_type | default("log") }} + # Paths that should be crawled and fetched + {% if path %}paths: + - {{ path }}{% endif %} + # Type of the files. Annotated in every documented + scan_frequency: {{scan_frequency | default("0.1s") }} + ignore_older: {{ignoreOlder}} + close_older: {{closeOlder}} + harvester_buffer_size: + encoding: {{encoding | default("utf-8") }} + tail_files: {{tailFiles}} + backoff: 0.1s + backoff_factor: 1 + max_backoff: 0.1s + force_close_files: {{force_close_files}} + + {% if fields %} + fields: + {% for k,v in fields.items() %} + {{k}}: {{v}} + {% endfor %} + {% endif %} + + fields_under_root: {{"true" if fieldsUnderRoot else "false"}} + + {% if include_lines %} + include_lines: {{include_lines}} + {% endif %} + + {% if exclude_lines %} + exclude_lines: {{exclude_lines}} + {% endif %} + + {% if exclude_files %} + exclude_files: {{exclude_files}} + {% endif %} + + max_bytes: {{ max_bytes|default(10485760) }} + + {% if json %} + json: + {% if json.message_key %}message_key: {{json.message_key}}{% endif %} + {% if json.keys_under_root %}keys_under_root: true{% endif %} + {% if json.overwrite_keys %}overwrite_keys: true{% endif %} + {% if json.add_error_key %}add_error_key: true{% endif %} + {% endif %} + + {% if multiline %} + multiline: + pattern: {{pattern}} + negate: {{negate}} + match: {{match}} + timeout: 1s + max_lines: {{ max_lines|default(500) }} + {% endif %} +{% endif %} + +filebeat.spool_size: +filebeat.idle_timeout: 0.1s +{% if not skip_registry_config %} +filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} +{%endif%} + + +#================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -name: {{shipperName}} +name: {{shipper_name}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. tags: [ -{%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} -{%- endif -%}] - + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] # Uncomment the following if you want to ignore transactions created # by the server on which the shipper is installed. This option is useful # to remove duplicates if shippers are installed on multiple servers. # ignore_outgoing: true +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} -############################# Output ############################################ - -# Configure what outputs to use when sending the data collected by filebeat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - # Elasticsearch as output - # Options: - # host, port: where Elasticsearch is listening on - # save_topology: specify if the topology is saved in Elasticsearch - #elasticsearch: - # enabled: false - # host: localhost - # port: 9200 - # save_topology: true - - # Redis as output - # Options: - # host, port: where Redis is listening on - # save_topology: specify if the topology is saved in Redis - #redis: - # enabled: false - # host: localhost - # port: 6379 - # save_topology: true - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("filebeat") }}" - rotate_every_kb: 1000 - number_of_files: 2 - -############################# Filters ######################################### {%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== filters: {%- if include_fields %} @@ -156,9 +128,20 @@ filters: {%- endif %} +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("filebeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + {% if path_data %} +#================================ Paths ===================================== path: data: {{path_data}} {%endif%} - -# vim: set ft=jinja: diff --git a/filebeat/tests/system/config/filebeat_prospectors.yml.j2 b/filebeat/tests/system/config/filebeat_prospectors.yml.j2 index c21b4df5b81c..53cd3ec90c5d 100644 --- a/filebeat/tests/system/config/filebeat_prospectors.yml.j2 +++ b/filebeat/tests/system/config/filebeat_prospectors.yml.j2 @@ -1,20 +1,15 @@ -filebeat: - prospectors: - {% for prospector in prospectors %} - - - paths: - - {{prospector.path}} - input: log - scan_frequency: 0.5s - encoding: {{prospector.encoding | default("plain") }} - {% endfor %} - idle_timeout: 0.5s - registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} +filebeat.prospectors: +{% for prospector in prospectors %} +- paths: + - {{prospector.path}} + scan_frequency: 0.5s + encoding: {{prospector.encoding | default("plain") }} +{% endfor %} +filebeat.idle_timeout: 0.5s +filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} -output: - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("filebeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("filebeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 diff --git a/filebeat/tests/system/test_fields.py b/filebeat/tests/system/test_fields.py index f58bf4226422..632cb4aa3840 100644 --- a/filebeat/tests/system/test_fields.py +++ b/filebeat/tests/system/test_fields.py @@ -67,7 +67,7 @@ def test_beat_fields(self): """ self.render_config_template( path=os.path.abspath(self.working_dir) + "/test.log", - shipperName="testShipperName" + shipper_name="testShipperName" ) with open(self.working_dir + "/test.log", "w") as f: diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index dd350e9e6958..2ca28abbc3d4 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -1,93 +1,142 @@ -metricbeat: - modules: - {% for m in modules -%} - - module: {{ m.name }} - {% if m.enabled is defined -%} - enabled: {{ m.enabled }} - {% endif -%} - - {% if m.hosts -%} - hosts: - {% for host in m.hosts -%} - - '{{ host }}' - {% endfor %} - {% endif -%} - - {% if m.metricsets -%} - metricsets: - {% for ms in m.metricsets -%} - - {{ ms }} - {% endfor %} - {% endif -%} - - {% if m.period -%} - period: {{ m.period }} - {% endif -%} - - {% if m.timeout -%} - timeout: {{ m.timeout }} - {% endif -%} - - {% if m.filters is defined -%} - filters: - {% for f in m.filters -%} - {% for k, v in f.items() -%} - - {{ k }}.fields: [ - {%- for field in v -%} - '{{ field }}' {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] - {%- endfor %} - {%- endfor %} - {% endif -%} - - {% if m.tags is defined -%} - tags: - {% for tag in m.tags -%} - - {{ tag }} - {% endfor %} - {% endif -%} - - {% if m.fields is defined -%} - {% if m.fields_under_root %}fields_under_root: true{% endif %} - fields: - {% for k, v in m.fields.items() -%} - {{ k }}: {{ v }} - {%- endfor %} - {% endif -%} - - {% if m.extras -%} - {% for k, v in m.extras.items() -%} - {{ k }}: {{ v }} - {% endfor %} - {% endif -%} +###################### Metricbeat Configuration Template ####################### + +metricbeat.modules: +{% for m in modules -%} +- module: {{ m.name }} + {% if m.enabled is defined -%} + enabled: {{ m.enabled }} + {% endif -%} + + {% if m.hosts -%} + hosts: + {% for host in m.hosts -%} + - '{{ host }}' + {% endfor %} + {% endif -%} + + {% if m.metricsets -%} + metricsets: + {% for ms in m.metricsets -%} + - {{ ms }} + {% endfor %} + {% endif -%} + + {% if m.period -%} + period: {{ m.period }} + {% endif -%} + + {% if m.timeout -%} + timeout: {{ m.timeout }} + {% endif -%} + + {% if m.filters is defined -%} + filters: + {% for f in m.filters -%} + {% for k, v in f.items() -%} + - {{ k }}.fields: [ + {%- for field in v -%} + '{{ field }}' {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endfor %} + {%- endfor %} + {% endif -%} + + {% if m.tags is defined -%} + tags: + {% for tag in m.tags -%} + - {{ tag }} + {% endfor %} + {% endif -%} + + {% if m.fields is defined -%} + {% if m.fields_under_root %}fields_under_root: true{% endif %} + fields: + {% for k, v in m.fields.items() -%} + {{ k }}: {{ v }} + {%- endfor %} + {% endif -%} + + {% if m.extras -%} + {% for k, v in m.extras.items() -%} + {{ k }}: {{ v }} + {% endfor %} + {% endif -%} +{%- endfor %} + +#================================ General ===================================== -output: - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("metricbeat") }}" - rotate_every_kb: 1000 +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: {{shipper_name}} -{%- if include_fields or drop_fields or drop_event %} +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +# ignore_outgoing: true + +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} + +{%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== filters: {%- if include_fields %} - include_fields: - {{ include_fields.condition | default() }} - fields: {{ include_fields.fields }} + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} {%- endif %} {%- if drop_fields %} - drop_fields: - {{ drop_fields.condition | default() }} - fields: {{ drop_fields.fields | default([]) }} + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} {%- endif %} + {%- if drop_event %} - drop_event: - {{ drop_event.condition | default() }} + {{ drop_event.condition | default()}} {%- endif %} + {%- endif %} -# vim: set ft=jinja: + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("metricbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + +{% if path_data %} +#================================ Paths ===================================== +path: + data: {{path_data}} +{%endif%} diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index bdddcfee7d04..1b028cecb8f0 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -1,132 +1,126 @@ -############################# Packetbeat ######################################### +#################### Packetbeat Configuration Template ######################### # Select the network interfaces to sniff the data. You can use the "any" # keyword to sniff on all connected interfaces. -packetbeat.interfaces: - device: {{ iface_device|default("any") }} +packetbeat.interfaces.device: {{ iface_device|default("any") }} {% if flows %} -packetbeat.flows: - period: -1s - timeout: 10s +#================================== Flows ===================================== +# Set network flow timeout. Flow is killed if no packet is received before being +# timed out. +packetbeat.flows.timeout: 10s + +# Configure reporting period. If set to -1, only killed flows will be reported +packetbeat.flows.period: -1s {% endif %} # Configure which protocols to monitor and the ports where they are # running. You can disable a given protocol by commenting out its # configuration. -packetbeat.protocols: - icmp: - enabled: true -{% if icmp_send_request %} send_request: true{%- endif %} -{% if icmp_send_response %} send_response: true{%- endif %} - - dns: - ports: [{{ dns_ports|default([53])|join(", ") }}] -{% if dns_include_authorities %} include_authorities: true{%- endif %} -{% if dns_include_additionals %} include_additionals: true{%- endif %} -{% if dns_send_request %} send_request: true{%- endif %} -{% if dns_send_response %} send_response: true{%- endif %} - - amqp: - ports: [{{ amqp_ports|default([5672])|join(", ") }}] -{% if amqp_send_request %} send_request: true{%- endif %} -{% if amqp_send_response %} send_response: true{%- endif %} - - http: - ports: [{{ http_ports|default([80])|join(", ") }}] -{% if http_send_request %} send_request: true{%- endif %} -{% if http_send_response %} send_response: true{%- endif %} -{% if http_send_all_headers %} send_all_headers: true{%- endif %} -{% if http_split_cookie %} split_cookie: true{%- endif %} +packetbeat.protocols.icmp: + enabled: true +{% if icmp_send_request %} send_request: true{%- endif %} +{% if icmp_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.dns: + ports: [{{ dns_ports|default([53])|join(", ") }}] +{% if dns_include_authorities %} include_authorities: true{%- endif %} +{% if dns_include_additionals %} include_additionals: true{%- endif %} +{% if dns_send_request %} send_request: true{%- endif %} +{% if dns_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.amqp: + ports: [{{ amqp_ports|default([5672])|join(", ") }}] +{% if amqp_send_request %} send_request: true{%- endif %} +{% if amqp_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.http: + ports: [{{ http_ports|default([80])|join(", ") }}] +{% if http_send_request %} send_request: true{%- endif %} +{% if http_send_response %} send_response: true{%- endif %} +{% if http_send_all_headers %} send_all_headers: true{%- endif %} +{% if http_split_cookie %} split_cookie: true{%- endif %} {% if http_send_headers %} - send_headers: [ - {%- for hdr in http_send_headers -%} - "{{ hdr }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + send_headers: [ + {%- for hdr in http_send_headers -%} + "{{ hdr }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if http_real_ip_header %} real_ip_header: "{{ http_real_ip_header }}"{% endif %} +{% if http_real_ip_header %} real_ip_header: "{{ http_real_ip_header }}"{% endif %} {%- if http_include_body_for %} - include_body_for: [ - {%- for ct in http_include_body_for -%} - "{{ ct }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + include_body_for: [ + {%- for ct in http_include_body_for -%} + "{{ ct }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if http_redact_authorization %} redact_authorization: true{% endif %} +{% if http_redact_authorization %} redact_authorization: true{% endif %} {%- if http_hide_keywords %} - hide_keywords: [ - {%- for keyword in http_hide_keywords -%} - "{{keyword}}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + hide_keywords: [ + {%- for keyword in http_hide_keywords -%} + "{{keyword}}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} - memcache: - ports: [{{ memcache_ports|default([11211])|join(", ") }}] -{% if memcache_send_request %} send_request: true{%- endif %} -{% if memcache_send_response %} send_response: true{%- endif %} -{% if memcache_parse_unknown %} parseunknown: true{%- endif %} -{% if memcache_max_values %} maxvalues: {{ memcache_max_values }}{%- endif %} -{% if memcache_udp_transaction_timeout %} udptransactiontimeout: {{ memcache_udp_transaction_timeout}}ms {%- endif %} - - mysql: - ports: [{{ mysql_ports|default([3306])|join(", ") }}] -{% if mysql_max_rows %} max_rows: {{mysql_max_rows}}{%- endif %} -{% if mysql_max_row_length %} max_row_length: {{mysql_max_row_length}}{%- endif %} -{% if mysql_send_request %} send_request: true{%- endif %} -{% if mysql_send_response %} send_response: true{%- endif %} - - pgsql: - ports: [{{ pgsql_ports|default([5432])|join(", ") }}] -{% if pgsql_max_rows %} max_rows: {{pgsql_max_rows}}{%- endif %} -{% if pgsql_max_row_length %} max_row_length: {{pgsql_max_row_length}}{%- endif %} -{% if pgsql_send_request %} send_request: true{%- endif %} -{% if pgsql_send_response %} send_response: true{%- endif %} - - redis: - ports: [{{ redis_ports|default([6379])|join(", ") }}] -{% if redis_send_request %} send_request: true{% endif %} -{% if redis_send_response %} send_response: true{% endif %} - - nfs: - ports: [{{ nfs_ports|default([2049])|join(", ") }}] - - thrift: - ports: [{{ thrift_ports|default([9090])|join(", ") }}] - transport_type: "{{ thrift_transport_type|default('socket') }}" +packetbeat.protocols.memcache: + ports: [{{ memcache_ports|default([11211])|join(", ") }}] +{% if memcache_send_request %} send_request: true{%- endif %} +{% if memcache_send_response %} send_response: true{%- endif %} +{% if memcache_parse_unknown %} parseunknown: true{%- endif %} +{% if memcache_max_values %} maxvalues: {{ memcache_max_values }}{%- endif %} +{% if memcache_udp_transaction_timeout %} udptransactiontimeout: {{ memcache_udp_transaction_timeout}}ms {%- endif %} + +packetbeat.protocols.mysql: + ports: [{{ mysql_ports|default([3306])|join(", ") }}] +{% if mysql_max_rows %} max_rows: {{mysql_max_rows}}{%- endif %} +{% if mysql_max_row_length %} max_row_length: {{mysql_max_row_length}}{%- endif %} +{% if mysql_send_request %} send_request: true{%- endif %} +{% if mysql_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.pgsql: + ports: [{{ pgsql_ports|default([5432])|join(", ") }}] +{% if pgsql_max_rows %} max_rows: {{pgsql_max_rows}}{%- endif %} +{% if pgsql_max_row_length %} max_row_length: {{pgsql_max_row_length}}{%- endif %} +{% if pgsql_send_request %} send_request: true{%- endif %} +{% if pgsql_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.redis: + ports: [{{ redis_ports|default([6379])|join(", ") }}] +{% if redis_send_request %} send_request: true{% endif %} +{% if redis_send_response %} send_response: true{% endif %} + +packetbeat.protocols.nfs: + ports: [{{ nfs_ports|default([2049])|join(", ") }}] + +packetbeat.protocols.thrift: + ports: [{{ thrift_ports|default([9090])|join(", ") }}] + transport_type: "{{ thrift_transport_type|default('socket') }}" {% if thrift_idl_files %} - idl_files: [ - {%- for file in thrift_idl_files -%} - "{{ beat.working_dir + '/' + file }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + idl_files: [ + {%- for file in thrift_idl_files -%} + "{{ beat.working_dir + '/' + file }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if thrift_send_request %} send_request: true{%- endif %} -{% if thrift_send_response %} send_response: true{%- endif %} +{% if thrift_send_request %} send_request: true{%- endif %} +{% if thrift_send_response %} send_response: true{%- endif %} - mongodb: - ports: [{{ mongodb_ports|default([27017])|join(", ") }}] -{% if mongodb_send_request %} send_request: true{%endif %} -{% if mongodb_send_response %} send_response: true{% endif %} -{% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} -{% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} +packetbeat.protocols.mongodb: + ports: [{{ mongodb_ports|default([27017])|join(", ") }}] +{% if mongodb_send_request %} send_request: true{%endif %} +{% if mongodb_send_response %} send_response: true{% endif %} +{% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} +{% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} -############################# Processes ####################################### {% if procs_enabled %} -# Configure the processes to be monitored and how to find them. If a process is -# monitored than Packetbeat attempts to use it's name to fill in the `proc` and -# `client_proc` fields. -# The processes can be found by searching their command line by a given string. -# -# Process matching is optional and can be enabled by uncommenting the following -# lines. -# +#=========================== Monitored processes ============================== packetbeat.procs: enabled: true monitored: @@ -146,38 +140,12 @@ packetbeat.procs: cmdline_grep: memcached {% endif %} +#================================ General ===================================== -############################# Filters ######################################### -{%- if drop_fields or drop_event or include_fields %} -filters: - - {%- if include_fields %} - - include_fields: - {{include_fields.condition | default()}} - fields: {{include_fields.fields | default([])}} - {%- endif %} - - {%- if drop_fields %} - - drop_fields: - {{drop_fields.condition | default()}} - fields: {{drop_fields.fields | default([])}} - {%- endif %} - - - {%- if drop_event %} - - drop_event: - {{ drop_event.condition | default()}} - {%- endif %} - -{%- endif %} - -# vim: set ft=jinja: - -############################# Shipper ######################################### # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -name: +name: {{shipperName}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different @@ -206,30 +174,39 @@ geoip: ] {%- endif %} -############################# Output ########################################## - -# Configure what outputs to use when sending the data collected by packetbeat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - # Elasticsearch as output - # Options: - # host, port: where Elasticsearch is listening on - # save_topology: specify if the topology is saved in Elasticsearch - #elasticsearch: - # host: localhost - # port: 9200 - # save_topology: true - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - file: - enabled: true - path: "{{ output_file_path|default(beat.working_dir + "/output") }}" - filename: "{{ output_file_filename|default("packetbeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 +{%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== +filters: + + {%- if include_fields %} + - include_fields: + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} + {%- endif %} + + {%- if drop_fields %} + - drop_fields: + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} + {%- endif %} + + + {%- if drop_event %} + - drop_event: + {{ drop_event.condition | default()}} + {%- endif %} + +{%- endif %} + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("packetbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 diff --git a/topbeat/tests/system/config/topbeat.yml.j2 b/topbeat/tests/system/config/topbeat.yml.j2 index 6e4bc4102aa1..2c4ddf9799da 100644 --- a/topbeat/tests/system/config/topbeat.yml.j2 +++ b/topbeat/tests/system/config/topbeat.yml.j2 @@ -1,122 +1,97 @@ ################### Topbeat Configuration Example ######################### ############################# Topbeat ############################################ -topbeat: - # In seconds, defines how often to read server statistics - period: {{ period | default(1) }} - - # Regular expression to match the processes that are monitored - # By default, all the processes are monitored - procs: [ - {%- for proc_pattern in proc_patterns|default([".*"]) -%} - "{{ proc_pattern}}" {%- if not loop.last %}, {% endif -%} - {%- endfor %}] - - # Statistics to collect (all enabled by default) - stats: - system: {{ "false" if system_stats == false else "true" }} - process: {{ "false" if process_stats == false else "true" }} - filesystem: {{ "false" if filesystem_stats == false else "true" }} - cpu_per_core: {{ "false" if cpu_per_core == false else "true" }} - -############################# Output ########################################## +# In seconds, defines how often to read server statistics +topbeat.period: {{ period | default(1) }} -# Configure what outputs to use when sending the data collected by the beat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - ### File as output - file: - # Enabling file output - enabled: true - - # Path to the directory where to save the generated files. The option is mandatory. - path: {{ output_file_path|default(beat.working_dir + "/output") }} - - - # Name of the generated files. The default is `topbeat` and it generates - # files: `topbeat`, `topbeat.1`, `topbeat.2`, etc. - filename: "{{ output_file_filename|default("topbeat") }}" - - # Maximum size in kilobytes of each file. When this size is reached, the files are - # rotated. The default value is 10 MB. - #rotate_every_kb: 10000 +# Regular expression to match the processes that are monitored +# By default, all the processes are monitored +topbeat.procs: [ +{%- for proc_pattern in proc_patterns|default([".*"]) -%} + "{{ proc_pattern}}" {%- if not loop.last %}, {% endif -%} +{%- endfor %}] - # Maximum number of files under path. When this number of files is reached, the - # oldest file is deleted and the rest are shifted from last to first. The default - # is 7 files. - #number_of_files: 7 +# Statistics to collect (all enabled by default) +topbeat.stats: + system: {{ "false" if system_stats == false else "true" }} + process: {{ "false" if process_stats == false else "true" }} + filesystem: {{ "false" if filesystem_stats == false else "true" }} + cpu_per_core: {{ "false" if cpu_per_core == false else "true" }} - -############################# Shipper ######################################### +#================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -#name: +name: {{shipper_name}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. -#tags: ["service-X", "web-tier"] +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] # Uncomment the following if you want to ignore transactions created # by the server on which the shipper is installed. This option is useful # to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - - -############################# Logging ######################################### - -#logging: - # Send all logging output to syslog. On Windows default is false, otherwise - # default is true. - #to_syslog: true - - # Write all logging output to files. Beats automatically rotate files if configurable - # limit is reached. - #to_files: false - - # Enable debug output for selected components. - #selectors: [] - - # Set log level - #level: error - - #files: - # The directory where the log files will written to. - #path: /var/log/mybeat - - # The name of the files where the logs are written to. - #name: mybeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - #keepfiles: 7 +# ignore_outgoing: true + +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} -############################# Filters ######################################### +{%- if drop_fields or drop_event or include_fields %} -{%- if include_fields or drop_fields or drop_event %} +#================================ Filters ===================================== filters: {%- if include_fields %} - include_fields: - {{ include_fields.condition | default() }} - fields: {{ include_fields.fields }} + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} {%- endif %} {%- if drop_fields %} - drop_fields: - {{ drop_fields.condition | default() }} - fields: {{ drop_fields.fields | default([]) }} + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} {%- endif %} + {%- if drop_event %} - drop_event: - {{ drop_event.condition | default() }} + {{ drop_event.condition | default()}} {%- endif %} + {%- endif %} + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("topbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + +{% if path_data %} +#================================ Paths ===================================== +path: + data: {{path_data}} +{%endif%} diff --git a/topbeat/tests/system/test_filesystem.py b/topbeat/tests/system/test_filesystem.py index f0ccd505dc7c..eea5623aeaa3 100644 --- a/topbeat/tests/system/test_filesystem.py +++ b/topbeat/tests/system/test_filesystem.py @@ -18,7 +18,9 @@ def test_filesystems(self): filesystem_stats=True ) topbeat = self.start_beat() - self.wait_until(lambda: self.log_contains(msg="output worker: publish")) + self.wait_until( + lambda: self.output_count(lambda x: x >= 1), + max_timeout=15) topbeat.check_kill_and_wait() output = self.read_output()[0] diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2 index fe308291d73d..f221376cbffc 100644 --- a/winlogbeat/tests/system/config/winlogbeat.yml.j2 +++ b/winlogbeat/tests/system/config/winlogbeat.yml.j2 @@ -1,52 +1,51 @@ -############################################################################### ############################# Winlogbeat ###################################### -winlogbeat: - {%- if event_logs %} - event_logs: - {% for log in event_logs -%} - - name: {{ log.name }} - {%- if log.api is defined %} - api: {{ log.api }} - {% endif %} - {%- if log.ignore_older is defined %} - ignore_older: {{ log.ignore_older }} - {% endif %} - {%- if log.event_id is defined %} - event_id: {{ log.event_id }} - {% endif %} - {%- if log.level is defined %} - level: {{ log.level }} - {% endif %} - {%- if log.provider %} - provider: - {% for p in log.provider -%} - - {{ p }} - {% endfor -%} - {% endif -%} - {%- if log.include_message is defined %} - include_message: {{ log.include_message }} - {% endif %} - {%- if log.include_xml is defined %} - include_xml: {{ log.include_xml }} - {% endif %} - {%- if log.tags is defined %} - tags: - {% for tag in log.tags -%} - - {{ tag }} - {% endfor -%} - {% endif -%} - {%- if log.fields is defined %} - {% if log.fields_under_root %}fields_under_root: true{% endif %} - fields: - {% for k, v in log.fields.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {% endif %} - {%- if log.invalid is defined %} - invalid: {{ log.invalid }} - {% endif %} - {% endfor -%} - {% endif %} + +{%- if event_logs %} +winlogbeat.event_logs: + {% for log in event_logs -%} + - name: {{ log.name }} + {%- if log.api is defined %} + api: {{ log.api }} + {% endif %} + {%- if log.ignore_older is defined %} + ignore_older: {{ log.ignore_older }} + {% endif %} + {%- if log.event_id is defined %} + event_id: {{ log.event_id }} + {% endif %} + {%- if log.level is defined %} + level: {{ log.level }} + {% endif %} + {%- if log.provider %} + provider: + {% for p in log.provider -%} + - {{ p }} + {% endfor -%} + {% endif -%} + {%- if log.include_message is defined %} + include_message: {{ log.include_message }} + {% endif %} + {%- if log.include_xml is defined %} + include_xml: {{ log.include_xml }} + {% endif %} + {%- if log.tags is defined %} + tags: + {% for tag in log.tags -%} + - {{ tag }} + {% endfor -%} + {% endif -%} + {%- if log.fields is defined %} + {% if log.fields_under_root %}fields_under_root: true{% endif %} + fields: + {% for k, v in log.fields.items() -%} + {{ k }}: {{ v }} + {% endfor -%} + {% endif %} + {%- if log.invalid is defined %} + invalid: {{ log.invalid }} + {% endif %} + {% endfor -%} +{% endif %} ############################################################################### ############################# Libbeat Config ################################## @@ -54,16 +53,11 @@ winlogbeat: ############################# Output ########################################## -output: - file: - path: {{ output_file_path | default(beat.working_dir + "/output") }} - {% if output_file_filename %} - filename: "{{ output_file_filename }}" - {% endif %} - rotate_every_kb: 1000 - -############################# Shipper ######################################### +#================================ General ===================================== +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. {%- if shipper_name %} name: {{ shipper_name }} {% endif %} @@ -83,4 +77,16 @@ fields: {% endfor -%} {% endif %} -# vim: set ft=jinja: +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path | default(beat.working_dir + "/output") }} + {% if output_file_filename %} + filename: "{{ output_file_filename }}" + {% endif %} + rotate_every_kb: 1000 +