Skip to content

Commit

Permalink
Updated configuration templates used in tests
Browse files Browse the repository at this point in the history 
The goal was to bring them closer to the new conventions we
use in the configuration files, so what we test is closer to what
we ship in terms of configuration.

I also made steps in unifying the libbeat portion of these templates so
we could remove the duplication in the future.

Part of #1417.
  • Loading branch information
Tudor Golubenco committed May 20, 2016
1 parent 7ee8ec7 commit dd69284
Show file tree
Hide file tree
Showing 8 changed files with 511 additions and 524 deletions.
225 changes: 104 additions & 121 deletions filebeat/tests/system/config/filebeat.yml.j2
View file
Original file line number Diff line number Diff line change
@@ -1,139 +1,111 @@
############################# Filebeat ######################################
filebeat:
prospectors:
{% if prospectors is not defined %}
{% set prospectors = true %}
{% endif %}
{% if prospectors %}
- input_type: {{input_type | default("log") }}
# Paths that should be crawled and fetched
{% if path %}paths:
- {{ path }}{% endif %}
# Type of the files. Annotated in every documented
scan_frequency: {{scan_frequency | default("0.1s") }}
ignore_older: {{ignoreOlder}}
close_older: {{closeOlder}}
harvester_buffer_size:
encoding: {{encoding | default("utf-8") }}
tail_files: {{tailFiles}}
backoff: 0.1s
backoff_factor: 1
max_backoff: 0.1s
force_close_files: {{force_close_files}}

{% if fields %}
fields:
{% for k,v in fields.items() %}
{{k}}: {{v}}
{% endfor %}
{% endif %}

fields_under_root: {{"true" if fieldsUnderRoot else "false"}}

{% if include_lines %}
include_lines: {{include_lines}}
{% endif %}

{% if exclude_lines %}
exclude_lines: {{exclude_lines}}
{% endif %}

{% if exclude_files %}
exclude_files: {{exclude_files}}
{% endif %}

max_bytes: {{ max_bytes|default(10485760) }}

{% if json %}
json:
{% if json.message_key %}message_key: {{json.message_key}}{% endif %}
{% if json.keys_under_root %}keys_under_root: true{% endif %}
{% if json.overwrite_keys %}overwrite_keys: true{% endif %}
{% if json.add_error_key %}add_error_key: true{% endif %}
{% endif %}

{% if multiline %}
multiline:
pattern: {{pattern}}
negate: {{negate}}
match: {{match}}
timeout: 1s
max_lines: {{ max_lines|default(500) }}
{% endif %}
{% endif %}
spool_size:
idle_timeout: 0.1s
{% if not skip_registry_config %}
registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}}
{%endif%}


############################# Shipper ############################################
###################### Filebeat Config Template ###############################

filebeat.prospectors:
{% if prospectors is not defined %}
{% set prospectors = true %}
{% endif %}
{% if prospectors %}
- input_type: {{input_type | default("log") }}
# Paths that should be crawled and fetched
{% if path %}paths:
- {{ path }}{% endif %}
# Type of the files. Annotated in every documented
scan_frequency: {{scan_frequency | default("0.1s") }}
ignore_older: {{ignoreOlder}}
close_older: {{closeOlder}}
harvester_buffer_size:
encoding: {{encoding | default("utf-8") }}
tail_files: {{tailFiles}}
backoff: 0.1s
backoff_factor: 1
max_backoff: 0.1s
force_close_files: {{force_close_files}}

{% if fields %}
fields:
{% for k,v in fields.items() %}
{{k}}: {{v}}
{% endfor %}
{% endif %}

fields_under_root: {{"true" if fieldsUnderRoot else "false"}}

{% if include_lines %}
include_lines: {{include_lines}}
{% endif %}

{% if exclude_lines %}
exclude_lines: {{exclude_lines}}
{% endif %}

{% if exclude_files %}
exclude_files: {{exclude_files}}
{% endif %}

max_bytes: {{ max_bytes|default(10485760) }}

{% if json %}
json:
{% if json.message_key %}message_key: {{json.message_key}}{% endif %}
{% if json.keys_under_root %}keys_under_root: true{% endif %}
{% if json.overwrite_keys %}overwrite_keys: true{% endif %}
{% if json.add_error_key %}add_error_key: true{% endif %}
{% endif %}

{% if multiline %}
multiline:
pattern: {{pattern}}
negate: {{negate}}
match: {{match}}
timeout: 1s
max_lines: {{ max_lines|default(500) }}
{% endif %}
{% endif %}

filebeat.spool_size:
filebeat.idle_timeout: 0.1s
{% if not skip_registry_config %}
filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}}
{%endif%}


#================================ General =====================================

# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
# If this options is not defined, the hostname is used.
name: {{shipperName}}
name: {{shipper_name}}

# The tags of the shipper are included in their own field with each
# transaction published. Tags make it easy to group servers by different
# logical properties.
tags: [
{%- if agent_tags -%}
{%- for tag in agent_tags -%}
"{{ tag }}"
{%- if not loop.last %}, {% endif -%}
{%- endfor -%}
{%- endif -%}]

{%- if agent_tags -%}
{%- for tag in agent_tags -%}
"{{ tag }}"
{%- if not loop.last %}, {% endif -%}
{%- endfor -%}
{%- endif -%}
]

# Uncomment the following if you want to ignore transactions created
# by the server on which the shipper is installed. This option is useful
# to remove duplicates if shippers are installed on multiple servers.
# ignore_outgoing: true

{% if geoip_paths is not none %}
geoip:
paths: [
{%- for path in geoip_paths -%}
"{{ beat.working_dir + '/' + path }}"
{%- if not loop.last %}, {% endif -%}
{%- endfor -%}
]
{%- endif %}

############################# Output ############################################

# Configure what outputs to use when sending the data collected by filebeat.
# You can enable one or multiple outputs by setting enabled option to true.
output:

# Elasticsearch as output
# Options:
# host, port: where Elasticsearch is listening on
# save_topology: specify if the topology is saved in Elasticsearch
#elasticsearch:
# enabled: false
# host: localhost
# port: 9200
# save_topology: true

# Redis as output
# Options:
# host, port: where Redis is listening on
# save_topology: specify if the topology is saved in Redis
#redis:
# enabled: false
# host: localhost
# port: 6379
# save_topology: true

# File as output
# Options
# path: where to save the files
# filename: name of the files
# rotate_every_kb: maximum size of the files in path
# number of files: maximum number of files in path
file:
enabled: true
path: {{ output_file_path|default(beat.working_dir + "/output") }}
filename: "{{ output_file_filename|default("filebeat") }}"
rotate_every_kb: 1000
number_of_files: 2

############################# Filters #########################################
{%- if drop_fields or drop_event or include_fields %}

#================================ Filters =====================================
filters:

{%- if include_fields %}
Expand All @@ -156,9 +128,20 @@ filters:

{%- endif %}

#================================ Outputs =====================================

# Configure what outputs to use when sending the data collected by the beat.
# Multiple outputs may be used.

#------------------------------- File output ----------------------------------
output.file:
path: {{ output_file_path|default(beat.working_dir + "/output") }}
filename: "{{ output_file_filename|default("filebeat") }}"
rotate_every_kb: 1000
#number_of_files: 7

{% if path_data %}
#================================ Paths =====================================
path:
data: {{path_data}}
{%endif%}

# vim: set ft=jinja:
33 changes: 14 additions & 19 deletions filebeat/tests/system/config/filebeat_prospectors.yml.j2
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,15 @@
filebeat:
prospectors:
{% for prospector in prospectors %}
-
paths:
- {{prospector.path}}
input: log
scan_frequency: 0.5s
encoding: {{prospector.encoding | default("plain") }}
{% endfor %}
idle_timeout: 0.5s
registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}}
filebeat.prospectors:
{% for prospector in prospectors %}
- paths:
- {{prospector.path}}
scan_frequency: 0.5s
encoding: {{prospector.encoding | default("plain") }}
{% endfor %}
filebeat.idle_timeout: 0.5s
filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}}

output:
file:
enabled: true
path: {{ output_file_path|default(beat.working_dir + "/output") }}
filename: "{{ output_file_filename|default("filebeat") }}"
rotate_every_kb: 1000
#number_of_files: 7
output.file:
path: {{ output_file_path|default(beat.working_dir + "/output") }}
filename: "{{ output_file_filename|default("filebeat") }}"
rotate_every_kb: 1000
#number_of_files: 7
2 changes: 1 addition & 1 deletion filebeat/tests/system/test_fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def test_beat_fields(self):
"""
self.render_config_template(
path=os.path.abspath(self.working_dir) + "/test.log",
shipperName="testShipperName"
shipper_name="testShipperName"
)

with open(self.working_dir + "/test.log", "w") as f:
Expand Down
Loading

0 comments on commit dd69284

Please sign in to comment.