diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index a3dd55880da..9dc76fb36b8 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -1,139 +1,111 @@ -############################# Filebeat ###################################### -filebeat: - prospectors: - {% if prospectors is not defined %} - {% set prospectors = true %} - {% endif %} - {% if prospectors %} - - input_type: {{input_type | default("log") }} - # Paths that should be crawled and fetched - {% if path %}paths: - - {{ path }}{% endif %} - # Type of the files. Annotated in every documented - scan_frequency: {{scan_frequency | default("0.1s") }} - ignore_older: {{ignoreOlder}} - close_older: {{closeOlder}} - harvester_buffer_size: - encoding: {{encoding | default("utf-8") }} - tail_files: {{tailFiles}} - backoff: 0.1s - backoff_factor: 1 - max_backoff: 0.1s - force_close_files: {{force_close_files}} - - {% if fields %} - fields: - {% for k,v in fields.items() %} - {{k}}: {{v}} - {% endfor %} - {% endif %} - - fields_under_root: {{"true" if fieldsUnderRoot else "false"}} - - {% if include_lines %} - include_lines: {{include_lines}} - {% endif %} - - {% if exclude_lines %} - exclude_lines: {{exclude_lines}} - {% endif %} - - {% if exclude_files %} - exclude_files: {{exclude_files}} - {% endif %} - - max_bytes: {{ max_bytes|default(10485760) }} - - {% if json %} - json: - {% if json.message_key %}message_key: {{json.message_key}}{% endif %} - {% if json.keys_under_root %}keys_under_root: true{% endif %} - {% if json.overwrite_keys %}overwrite_keys: true{% endif %} - {% if json.add_error_key %}add_error_key: true{% endif %} - {% endif %} - - {% if multiline %} - multiline: - pattern: {{pattern}} - negate: {{negate}} - match: {{match}} - timeout: 1s - max_lines: {{ max_lines|default(500) }} - {% endif %} - {% endif %} - spool_size: - idle_timeout: 0.1s - {% if not skip_registry_config %} - registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} - {%endif%} - - -############################# Shipper ############################################ +###################### Filebeat Config Template ############################### + +filebeat.prospectors: +{% if prospectors is not defined %} +{% set prospectors = true %} +{% endif %} +{% if prospectors %} +- input_type: {{input_type | default("log") }} + # Paths that should be crawled and fetched + {% if path %}paths: + - {{ path }}{% endif %} + # Type of the files. Annotated in every documented + scan_frequency: {{scan_frequency | default("0.1s") }} + ignore_older: {{ignoreOlder}} + close_older: {{closeOlder}} + harvester_buffer_size: + encoding: {{encoding | default("utf-8") }} + tail_files: {{tailFiles}} + backoff: 0.1s + backoff_factor: 1 + max_backoff: 0.1s + force_close_files: {{force_close_files}} + + {% if fields %} + fields: + {% for k,v in fields.items() %} + {{k}}: {{v}} + {% endfor %} + {% endif %} + + fields_under_root: {{"true" if fieldsUnderRoot else "false"}} + + {% if include_lines %} + include_lines: {{include_lines}} + {% endif %} + + {% if exclude_lines %} + exclude_lines: {{exclude_lines}} + {% endif %} + + {% if exclude_files %} + exclude_files: {{exclude_files}} + {% endif %} + + max_bytes: {{ max_bytes|default(10485760) }} + + {% if json %} + json: + {% if json.message_key %}message_key: {{json.message_key}}{% endif %} + {% if json.keys_under_root %}keys_under_root: true{% endif %} + {% if json.overwrite_keys %}overwrite_keys: true{% endif %} + {% if json.add_error_key %}add_error_key: true{% endif %} + {% endif %} + + {% if multiline %} + multiline: + pattern: {{pattern}} + negate: {{negate}} + match: {{match}} + timeout: 1s + max_lines: {{ max_lines|default(500) }} + {% endif %} +{% endif %} + +filebeat.spool_size: +filebeat.idle_timeout: 0.1s +{% if not skip_registry_config %} +filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} +{%endif%} + + +#================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -name: {{shipperName}} +name: {{shipper_name}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. tags: [ -{%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} -{%- endif -%}] - + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] # Uncomment the following if you want to ignore transactions created # by the server on which the shipper is installed. This option is useful # to remove duplicates if shippers are installed on multiple servers. # ignore_outgoing: true +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} -############################# Output ############################################ - -# Configure what outputs to use when sending the data collected by filebeat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - # Elasticsearch as output - # Options: - # host, port: where Elasticsearch is listening on - # save_topology: specify if the topology is saved in Elasticsearch - #elasticsearch: - # enabled: false - # host: localhost - # port: 9200 - # save_topology: true - - # Redis as output - # Options: - # host, port: where Redis is listening on - # save_topology: specify if the topology is saved in Redis - #redis: - # enabled: false - # host: localhost - # port: 6379 - # save_topology: true - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("filebeat") }}" - rotate_every_kb: 1000 - number_of_files: 2 - -############################# Filters ######################################### {%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== filters: {%- if include_fields %} @@ -156,9 +128,20 @@ filters: {%- endif %} +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("filebeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + {% if path_data %} +#================================ Paths ===================================== path: data: {{path_data}} {%endif%} - -# vim: set ft=jinja: diff --git a/filebeat/tests/system/config/filebeat_prospectors.yml.j2 b/filebeat/tests/system/config/filebeat_prospectors.yml.j2 index c21b4df5b81..53cd3ec90c5 100644 --- a/filebeat/tests/system/config/filebeat_prospectors.yml.j2 +++ b/filebeat/tests/system/config/filebeat_prospectors.yml.j2 @@ -1,20 +1,15 @@ -filebeat: - prospectors: - {% for prospector in prospectors %} - - - paths: - - {{prospector.path}} - input: log - scan_frequency: 0.5s - encoding: {{prospector.encoding | default("plain") }} - {% endfor %} - idle_timeout: 0.5s - registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} +filebeat.prospectors: +{% for prospector in prospectors %} +- paths: + - {{prospector.path}} + scan_frequency: 0.5s + encoding: {{prospector.encoding | default("plain") }} +{% endfor %} +filebeat.idle_timeout: 0.5s +filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}} -output: - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("filebeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("filebeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 diff --git a/filebeat/tests/system/test_fields.py b/filebeat/tests/system/test_fields.py index f58bf422642..632cb4aa384 100644 --- a/filebeat/tests/system/test_fields.py +++ b/filebeat/tests/system/test_fields.py @@ -67,7 +67,7 @@ def test_beat_fields(self): """ self.render_config_template( path=os.path.abspath(self.working_dir) + "/test.log", - shipperName="testShipperName" + shipper_name="testShipperName" ) with open(self.working_dir + "/test.log", "w") as f: diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index dd350e9e695..2ca28abbc3d 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -1,93 +1,142 @@ -metricbeat: - modules: - {% for m in modules -%} - - module: {{ m.name }} - {% if m.enabled is defined -%} - enabled: {{ m.enabled }} - {% endif -%} - - {% if m.hosts -%} - hosts: - {% for host in m.hosts -%} - - '{{ host }}' - {% endfor %} - {% endif -%} - - {% if m.metricsets -%} - metricsets: - {% for ms in m.metricsets -%} - - {{ ms }} - {% endfor %} - {% endif -%} - - {% if m.period -%} - period: {{ m.period }} - {% endif -%} - - {% if m.timeout -%} - timeout: {{ m.timeout }} - {% endif -%} - - {% if m.filters is defined -%} - filters: - {% for f in m.filters -%} - {% for k, v in f.items() -%} - - {{ k }}.fields: [ - {%- for field in v -%} - '{{ field }}' {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] - {%- endfor %} - {%- endfor %} - {% endif -%} - - {% if m.tags is defined -%} - tags: - {% for tag in m.tags -%} - - {{ tag }} - {% endfor %} - {% endif -%} - - {% if m.fields is defined -%} - {% if m.fields_under_root %}fields_under_root: true{% endif %} - fields: - {% for k, v in m.fields.items() -%} - {{ k }}: {{ v }} - {%- endfor %} - {% endif -%} - - {% if m.extras -%} - {% for k, v in m.extras.items() -%} - {{ k }}: {{ v }} - {% endfor %} - {% endif -%} +###################### Metricbeat Configuration Template ####################### + +metricbeat.modules: +{% for m in modules -%} +- module: {{ m.name }} + {% if m.enabled is defined -%} + enabled: {{ m.enabled }} + {% endif -%} + + {% if m.hosts -%} + hosts: + {% for host in m.hosts -%} + - '{{ host }}' + {% endfor %} + {% endif -%} + + {% if m.metricsets -%} + metricsets: + {% for ms in m.metricsets -%} + - {{ ms }} + {% endfor %} + {% endif -%} + + {% if m.period -%} + period: {{ m.period }} + {% endif -%} + + {% if m.timeout -%} + timeout: {{ m.timeout }} + {% endif -%} + + {% if m.filters is defined -%} + filters: + {% for f in m.filters -%} + {% for k, v in f.items() -%} + - {{ k }}.fields: [ + {%- for field in v -%} + '{{ field }}' {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endfor %} + {%- endfor %} + {% endif -%} + + {% if m.tags is defined -%} + tags: + {% for tag in m.tags -%} + - {{ tag }} + {% endfor %} + {% endif -%} + + {% if m.fields is defined -%} + {% if m.fields_under_root %}fields_under_root: true{% endif %} + fields: + {% for k, v in m.fields.items() -%} + {{ k }}: {{ v }} + {%- endfor %} + {% endif -%} + + {% if m.extras -%} + {% for k, v in m.extras.items() -%} + {{ k }}: {{ v }} + {% endfor %} + {% endif -%} +{%- endfor %} + +#================================ General ===================================== -output: - file: - enabled: true - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("metricbeat") }}" - rotate_every_kb: 1000 +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: {{shipper_name}} -{%- if include_fields or drop_fields or drop_event %} +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +# ignore_outgoing: true + +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} + +{%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== filters: {%- if include_fields %} - include_fields: - {{ include_fields.condition | default() }} - fields: {{ include_fields.fields }} + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} {%- endif %} {%- if drop_fields %} - drop_fields: - {{ drop_fields.condition | default() }} - fields: {{ drop_fields.fields | default([]) }} + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} {%- endif %} + {%- if drop_event %} - drop_event: - {{ drop_event.condition | default() }} + {{ drop_event.condition | default()}} {%- endif %} + {%- endif %} -# vim: set ft=jinja: + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("metricbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + +{% if path_data %} +#================================ Paths ===================================== +path: + data: {{path_data}} +{%endif%} diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index bdddcfee7d0..1b028cecb8f 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -1,132 +1,126 @@ -############################# Packetbeat ######################################### +#################### Packetbeat Configuration Template ######################### # Select the network interfaces to sniff the data. You can use the "any" # keyword to sniff on all connected interfaces. -packetbeat.interfaces: - device: {{ iface_device|default("any") }} +packetbeat.interfaces.device: {{ iface_device|default("any") }} {% if flows %} -packetbeat.flows: - period: -1s - timeout: 10s +#================================== Flows ===================================== +# Set network flow timeout. Flow is killed if no packet is received before being +# timed out. +packetbeat.flows.timeout: 10s + +# Configure reporting period. If set to -1, only killed flows will be reported +packetbeat.flows.period: -1s {% endif %} # Configure which protocols to monitor and the ports where they are # running. You can disable a given protocol by commenting out its # configuration. -packetbeat.protocols: - icmp: - enabled: true -{% if icmp_send_request %} send_request: true{%- endif %} -{% if icmp_send_response %} send_response: true{%- endif %} - - dns: - ports: [{{ dns_ports|default([53])|join(", ") }}] -{% if dns_include_authorities %} include_authorities: true{%- endif %} -{% if dns_include_additionals %} include_additionals: true{%- endif %} -{% if dns_send_request %} send_request: true{%- endif %} -{% if dns_send_response %} send_response: true{%- endif %} - - amqp: - ports: [{{ amqp_ports|default([5672])|join(", ") }}] -{% if amqp_send_request %} send_request: true{%- endif %} -{% if amqp_send_response %} send_response: true{%- endif %} - - http: - ports: [{{ http_ports|default([80])|join(", ") }}] -{% if http_send_request %} send_request: true{%- endif %} -{% if http_send_response %} send_response: true{%- endif %} -{% if http_send_all_headers %} send_all_headers: true{%- endif %} -{% if http_split_cookie %} split_cookie: true{%- endif %} +packetbeat.protocols.icmp: + enabled: true +{% if icmp_send_request %} send_request: true{%- endif %} +{% if icmp_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.dns: + ports: [{{ dns_ports|default([53])|join(", ") }}] +{% if dns_include_authorities %} include_authorities: true{%- endif %} +{% if dns_include_additionals %} include_additionals: true{%- endif %} +{% if dns_send_request %} send_request: true{%- endif %} +{% if dns_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.amqp: + ports: [{{ amqp_ports|default([5672])|join(", ") }}] +{% if amqp_send_request %} send_request: true{%- endif %} +{% if amqp_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.http: + ports: [{{ http_ports|default([80])|join(", ") }}] +{% if http_send_request %} send_request: true{%- endif %} +{% if http_send_response %} send_response: true{%- endif %} +{% if http_send_all_headers %} send_all_headers: true{%- endif %} +{% if http_split_cookie %} split_cookie: true{%- endif %} {% if http_send_headers %} - send_headers: [ - {%- for hdr in http_send_headers -%} - "{{ hdr }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + send_headers: [ + {%- for hdr in http_send_headers -%} + "{{ hdr }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if http_real_ip_header %} real_ip_header: "{{ http_real_ip_header }}"{% endif %} +{% if http_real_ip_header %} real_ip_header: "{{ http_real_ip_header }}"{% endif %} {%- if http_include_body_for %} - include_body_for: [ - {%- for ct in http_include_body_for -%} - "{{ ct }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + include_body_for: [ + {%- for ct in http_include_body_for -%} + "{{ ct }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if http_redact_authorization %} redact_authorization: true{% endif %} +{% if http_redact_authorization %} redact_authorization: true{% endif %} {%- if http_hide_keywords %} - hide_keywords: [ - {%- for keyword in http_hide_keywords -%} - "{{keyword}}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + hide_keywords: [ + {%- for keyword in http_hide_keywords -%} + "{{keyword}}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} - memcache: - ports: [{{ memcache_ports|default([11211])|join(", ") }}] -{% if memcache_send_request %} send_request: true{%- endif %} -{% if memcache_send_response %} send_response: true{%- endif %} -{% if memcache_parse_unknown %} parseunknown: true{%- endif %} -{% if memcache_max_values %} maxvalues: {{ memcache_max_values }}{%- endif %} -{% if memcache_udp_transaction_timeout %} udptransactiontimeout: {{ memcache_udp_transaction_timeout}}ms {%- endif %} - - mysql: - ports: [{{ mysql_ports|default([3306])|join(", ") }}] -{% if mysql_max_rows %} max_rows: {{mysql_max_rows}}{%- endif %} -{% if mysql_max_row_length %} max_row_length: {{mysql_max_row_length}}{%- endif %} -{% if mysql_send_request %} send_request: true{%- endif %} -{% if mysql_send_response %} send_response: true{%- endif %} - - pgsql: - ports: [{{ pgsql_ports|default([5432])|join(", ") }}] -{% if pgsql_max_rows %} max_rows: {{pgsql_max_rows}}{%- endif %} -{% if pgsql_max_row_length %} max_row_length: {{pgsql_max_row_length}}{%- endif %} -{% if pgsql_send_request %} send_request: true{%- endif %} -{% if pgsql_send_response %} send_response: true{%- endif %} - - redis: - ports: [{{ redis_ports|default([6379])|join(", ") }}] -{% if redis_send_request %} send_request: true{% endif %} -{% if redis_send_response %} send_response: true{% endif %} - - nfs: - ports: [{{ nfs_ports|default([2049])|join(", ") }}] - - thrift: - ports: [{{ thrift_ports|default([9090])|join(", ") }}] - transport_type: "{{ thrift_transport_type|default('socket') }}" +packetbeat.protocols.memcache: + ports: [{{ memcache_ports|default([11211])|join(", ") }}] +{% if memcache_send_request %} send_request: true{%- endif %} +{% if memcache_send_response %} send_response: true{%- endif %} +{% if memcache_parse_unknown %} parseunknown: true{%- endif %} +{% if memcache_max_values %} maxvalues: {{ memcache_max_values }}{%- endif %} +{% if memcache_udp_transaction_timeout %} udptransactiontimeout: {{ memcache_udp_transaction_timeout}}ms {%- endif %} + +packetbeat.protocols.mysql: + ports: [{{ mysql_ports|default([3306])|join(", ") }}] +{% if mysql_max_rows %} max_rows: {{mysql_max_rows}}{%- endif %} +{% if mysql_max_row_length %} max_row_length: {{mysql_max_row_length}}{%- endif %} +{% if mysql_send_request %} send_request: true{%- endif %} +{% if mysql_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.pgsql: + ports: [{{ pgsql_ports|default([5432])|join(", ") }}] +{% if pgsql_max_rows %} max_rows: {{pgsql_max_rows}}{%- endif %} +{% if pgsql_max_row_length %} max_row_length: {{pgsql_max_row_length}}{%- endif %} +{% if pgsql_send_request %} send_request: true{%- endif %} +{% if pgsql_send_response %} send_response: true{%- endif %} + +packetbeat.protocols.redis: + ports: [{{ redis_ports|default([6379])|join(", ") }}] +{% if redis_send_request %} send_request: true{% endif %} +{% if redis_send_response %} send_response: true{% endif %} + +packetbeat.protocols.nfs: + ports: [{{ nfs_ports|default([2049])|join(", ") }}] + +packetbeat.protocols.thrift: + ports: [{{ thrift_ports|default([9090])|join(", ") }}] + transport_type: "{{ thrift_transport_type|default('socket') }}" {% if thrift_idl_files %} - idl_files: [ - {%- for file in thrift_idl_files -%} - "{{ beat.working_dir + '/' + file }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] + idl_files: [ + {%- for file in thrift_idl_files -%} + "{{ beat.working_dir + '/' + file }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} -{% if thrift_send_request %} send_request: true{%- endif %} -{% if thrift_send_response %} send_response: true{%- endif %} +{% if thrift_send_request %} send_request: true{%- endif %} +{% if thrift_send_response %} send_response: true{%- endif %} - mongodb: - ports: [{{ mongodb_ports|default([27017])|join(", ") }}] -{% if mongodb_send_request %} send_request: true{%endif %} -{% if mongodb_send_response %} send_response: true{% endif %} -{% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} -{% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} +packetbeat.protocols.mongodb: + ports: [{{ mongodb_ports|default([27017])|join(", ") }}] +{% if mongodb_send_request %} send_request: true{%endif %} +{% if mongodb_send_response %} send_response: true{% endif %} +{% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} +{% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} -############################# Processes ####################################### {% if procs_enabled %} -# Configure the processes to be monitored and how to find them. If a process is -# monitored than Packetbeat attempts to use it's name to fill in the `proc` and -# `client_proc` fields. -# The processes can be found by searching their command line by a given string. -# -# Process matching is optional and can be enabled by uncommenting the following -# lines. -# +#=========================== Monitored processes ============================== packetbeat.procs: enabled: true monitored: @@ -146,38 +140,12 @@ packetbeat.procs: cmdline_grep: memcached {% endif %} +#================================ General ===================================== -############################# Filters ######################################### -{%- if drop_fields or drop_event or include_fields %} -filters: - - {%- if include_fields %} - - include_fields: - {{include_fields.condition | default()}} - fields: {{include_fields.fields | default([])}} - {%- endif %} - - {%- if drop_fields %} - - drop_fields: - {{drop_fields.condition | default()}} - fields: {{drop_fields.fields | default([])}} - {%- endif %} - - - {%- if drop_event %} - - drop_event: - {{ drop_event.condition | default()}} - {%- endif %} - -{%- endif %} - -# vim: set ft=jinja: - -############################# Shipper ######################################### # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -name: +name: {{shipperName}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different @@ -206,30 +174,39 @@ geoip: ] {%- endif %} -############################# Output ########################################## - -# Configure what outputs to use when sending the data collected by packetbeat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - # Elasticsearch as output - # Options: - # host, port: where Elasticsearch is listening on - # save_topology: specify if the topology is saved in Elasticsearch - #elasticsearch: - # host: localhost - # port: 9200 - # save_topology: true - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - file: - enabled: true - path: "{{ output_file_path|default(beat.working_dir + "/output") }}" - filename: "{{ output_file_filename|default("packetbeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 +{%- if drop_fields or drop_event or include_fields %} + +#================================ Filters ===================================== +filters: + + {%- if include_fields %} + - include_fields: + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} + {%- endif %} + + {%- if drop_fields %} + - drop_fields: + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} + {%- endif %} + + + {%- if drop_event %} + - drop_event: + {{ drop_event.condition | default()}} + {%- endif %} + +{%- endif %} + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("packetbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 diff --git a/topbeat/tests/system/config/topbeat.yml.j2 b/topbeat/tests/system/config/topbeat.yml.j2 index 6e4bc4102aa..2c4ddf9799d 100644 --- a/topbeat/tests/system/config/topbeat.yml.j2 +++ b/topbeat/tests/system/config/topbeat.yml.j2 @@ -1,122 +1,97 @@ ################### Topbeat Configuration Example ######################### ############################# Topbeat ############################################ -topbeat: - # In seconds, defines how often to read server statistics - period: {{ period | default(1) }} - - # Regular expression to match the processes that are monitored - # By default, all the processes are monitored - procs: [ - {%- for proc_pattern in proc_patterns|default([".*"]) -%} - "{{ proc_pattern}}" {%- if not loop.last %}, {% endif -%} - {%- endfor %}] - - # Statistics to collect (all enabled by default) - stats: - system: {{ "false" if system_stats == false else "true" }} - process: {{ "false" if process_stats == false else "true" }} - filesystem: {{ "false" if filesystem_stats == false else "true" }} - cpu_per_core: {{ "false" if cpu_per_core == false else "true" }} - -############################# Output ########################################## +# In seconds, defines how often to read server statistics +topbeat.period: {{ period | default(1) }} -# Configure what outputs to use when sending the data collected by the beat. -# You can enable one or multiple outputs by setting enabled option to true. -output: - - ### File as output - file: - # Enabling file output - enabled: true - - # Path to the directory where to save the generated files. The option is mandatory. - path: {{ output_file_path|default(beat.working_dir + "/output") }} - - - # Name of the generated files. The default is `topbeat` and it generates - # files: `topbeat`, `topbeat.1`, `topbeat.2`, etc. - filename: "{{ output_file_filename|default("topbeat") }}" - - # Maximum size in kilobytes of each file. When this size is reached, the files are - # rotated. The default value is 10 MB. - #rotate_every_kb: 10000 +# Regular expression to match the processes that are monitored +# By default, all the processes are monitored +topbeat.procs: [ +{%- for proc_pattern in proc_patterns|default([".*"]) -%} + "{{ proc_pattern}}" {%- if not loop.last %}, {% endif -%} +{%- endfor %}] - # Maximum number of files under path. When this number of files is reached, the - # oldest file is deleted and the rest are shifted from last to first. The default - # is 7 files. - #number_of_files: 7 +# Statistics to collect (all enabled by default) +topbeat.stats: + system: {{ "false" if system_stats == false else "true" }} + process: {{ "false" if process_stats == false else "true" }} + filesystem: {{ "false" if filesystem_stats == false else "true" }} + cpu_per_core: {{ "false" if cpu_per_core == false else "true" }} - -############################# Shipper ######################################### +#================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. -#name: +name: {{shipper_name}} # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. -#tags: ["service-X", "web-tier"] +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] # Uncomment the following if you want to ignore transactions created # by the server on which the shipper is installed. This option is useful # to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - - -############################# Logging ######################################### - -#logging: - # Send all logging output to syslog. On Windows default is false, otherwise - # default is true. - #to_syslog: true - - # Write all logging output to files. Beats automatically rotate files if configurable - # limit is reached. - #to_files: false - - # Enable debug output for selected components. - #selectors: [] - - # Set log level - #level: error - - #files: - # The directory where the log files will written to. - #path: /var/log/mybeat - - # The name of the files where the logs are written to. - #name: mybeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - #keepfiles: 7 +# ignore_outgoing: true + +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} -############################# Filters ######################################### +{%- if drop_fields or drop_event or include_fields %} -{%- if include_fields or drop_fields or drop_event %} +#================================ Filters ===================================== filters: {%- if include_fields %} - include_fields: - {{ include_fields.condition | default() }} - fields: {{ include_fields.fields }} + {{include_fields.condition | default()}} + fields: {{include_fields.fields | default([])}} {%- endif %} {%- if drop_fields %} - drop_fields: - {{ drop_fields.condition | default() }} - fields: {{ drop_fields.fields | default([]) }} + {{drop_fields.condition | default()}} + fields: {{drop_fields.fields | default([])}} {%- endif %} + {%- if drop_event %} - drop_event: - {{ drop_event.condition | default() }} + {{ drop_event.condition | default()}} {%- endif %} + {%- endif %} + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default("topbeat") }}" + rotate_every_kb: 1000 + #number_of_files: 7 + +{% if path_data %} +#================================ Paths ===================================== +path: + data: {{path_data}} +{%endif%} diff --git a/topbeat/tests/system/test_filesystem.py b/topbeat/tests/system/test_filesystem.py index f0ccd505dc7..eea5623aeaa 100644 --- a/topbeat/tests/system/test_filesystem.py +++ b/topbeat/tests/system/test_filesystem.py @@ -18,7 +18,9 @@ def test_filesystems(self): filesystem_stats=True ) topbeat = self.start_beat() - self.wait_until(lambda: self.log_contains(msg="output worker: publish")) + self.wait_until( + lambda: self.output_count(lambda x: x >= 1), + max_timeout=15) topbeat.check_kill_and_wait() output = self.read_output()[0] diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2 index fe308291d73..f221376cbff 100644 --- a/winlogbeat/tests/system/config/winlogbeat.yml.j2 +++ b/winlogbeat/tests/system/config/winlogbeat.yml.j2 @@ -1,52 +1,51 @@ -############################################################################### ############################# Winlogbeat ###################################### -winlogbeat: - {%- if event_logs %} - event_logs: - {% for log in event_logs -%} - - name: {{ log.name }} - {%- if log.api is defined %} - api: {{ log.api }} - {% endif %} - {%- if log.ignore_older is defined %} - ignore_older: {{ log.ignore_older }} - {% endif %} - {%- if log.event_id is defined %} - event_id: {{ log.event_id }} - {% endif %} - {%- if log.level is defined %} - level: {{ log.level }} - {% endif %} - {%- if log.provider %} - provider: - {% for p in log.provider -%} - - {{ p }} - {% endfor -%} - {% endif -%} - {%- if log.include_message is defined %} - include_message: {{ log.include_message }} - {% endif %} - {%- if log.include_xml is defined %} - include_xml: {{ log.include_xml }} - {% endif %} - {%- if log.tags is defined %} - tags: - {% for tag in log.tags -%} - - {{ tag }} - {% endfor -%} - {% endif -%} - {%- if log.fields is defined %} - {% if log.fields_under_root %}fields_under_root: true{% endif %} - fields: - {% for k, v in log.fields.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {% endif %} - {%- if log.invalid is defined %} - invalid: {{ log.invalid }} - {% endif %} - {% endfor -%} - {% endif %} + +{%- if event_logs %} +winlogbeat.event_logs: + {% for log in event_logs -%} + - name: {{ log.name }} + {%- if log.api is defined %} + api: {{ log.api }} + {% endif %} + {%- if log.ignore_older is defined %} + ignore_older: {{ log.ignore_older }} + {% endif %} + {%- if log.event_id is defined %} + event_id: {{ log.event_id }} + {% endif %} + {%- if log.level is defined %} + level: {{ log.level }} + {% endif %} + {%- if log.provider %} + provider: + {% for p in log.provider -%} + - {{ p }} + {% endfor -%} + {% endif -%} + {%- if log.include_message is defined %} + include_message: {{ log.include_message }} + {% endif %} + {%- if log.include_xml is defined %} + include_xml: {{ log.include_xml }} + {% endif %} + {%- if log.tags is defined %} + tags: + {% for tag in log.tags -%} + - {{ tag }} + {% endfor -%} + {% endif -%} + {%- if log.fields is defined %} + {% if log.fields_under_root %}fields_under_root: true{% endif %} + fields: + {% for k, v in log.fields.items() -%} + {{ k }}: {{ v }} + {% endfor -%} + {% endif %} + {%- if log.invalid is defined %} + invalid: {{ log.invalid }} + {% endif %} + {% endfor -%} +{% endif %} ############################################################################### ############################# Libbeat Config ################################## @@ -54,16 +53,11 @@ winlogbeat: ############################# Output ########################################## -output: - file: - path: {{ output_file_path | default(beat.working_dir + "/output") }} - {% if output_file_filename %} - filename: "{{ output_file_filename }}" - {% endif %} - rotate_every_kb: 1000 - -############################# Shipper ######################################### +#================================ General ===================================== +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. {%- if shipper_name %} name: {{ shipper_name }} {% endif %} @@ -83,4 +77,16 @@ fields: {% endfor -%} {% endif %} -# vim: set ft=jinja: +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#------------------------------- File output ---------------------------------- +output.file: + path: {{ output_file_path | default(beat.working_dir + "/output") }} + {% if output_file_filename %} + filename: "{{ output_file_filename }}" + {% endif %} + rotate_every_kb: 1000 +