-
Notifications
You must be signed in to change notification settings - Fork 4.9k
/
test_cmd_setup_index_management.py
243 lines (209 loc) · 11.2 KB
/
test_cmd_setup_index_management.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
import logging
import os
import pytest
import unittest
from base import BaseTest
from elasticsearch import RequestError
from idxmgmt import IdxMgmt
INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
class TestCommandSetupIndexManagement(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupIndexManagement, self).setUp()
self.cmd = "--index-management"
# auto-derived default settings, if nothing else is set
self.policy_name = self.beat_name
self.data_stream = self.beat_name + "-9.9.9"
self.custom_policy = self.beat_name + "_bar"
self.custom_template = self.beat_name + "_foobar"
self.es = self.es_client()
self.es = self.get_elasticsearch_instance()
self.idxmgmt = IdxMgmt(self.es, self.data_stream)
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
def render_config(self, **kwargs):
self.render_config_template(
# Note that the template is such that we need to pass in 'username' as opposed to 'user' and
# 'password' instead of 'pass'.
elasticsearch=self.get_elasticsearch_template_config(),
es_template_name=self.data_stream,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_index_template_index_pattern(self.data_stream, [self.data_stream])
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-e", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.template.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.data_stream+"ba")
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_policy_name(self):
"""
Test setup --index-management when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.policy_name=" + self.custom_policy])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_policy_no_overwrite(self):
"""
Test setup --index-management respects overwrite configuration
"""
policy_name = "mockbeat-test"
# update policy to verify overwrite behaviour
self.es.transport.perform_request('PUT', '/_ilm/policy/' + policy_name,
body={
"policy": {
"phases": {
"delete": {
"actions": {
"delete": {}
}
}
}
}
})
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is not overwritten
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=true",
"-E", "setup.ilm.overwrite=false",
"-E", "setup.ilm.policy_name=" + policy_name])
assert exit_code == 0
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is overwritten
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=true",
"-E", "setup.ilm.overwrite=true",
"-E", "setup.ilm.policy_name=" + policy_name])
assert exit_code == 0
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" not in resp[policy_name]["policy"]["phases"]
assert "hot" in resp[policy_name]["policy"]["phases"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_name_and_pattern_on_ilm_disabled(self):
"""
Test setup --index-management respects template.name and template.pattern when ilm is disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"])
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_with_opts(self):
"""
Test setup --index-management with config options
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_index_template/' + self.data_stream)
found = False
for index_template in resp["index_templates"]:
if self.data_stream == index_template["name"]:
found = True
index = index_template["index_template"]["template"]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
assert found
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_overwrite_template_on_ilm_policy_created(self):
"""
Test setup --index-management overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.priority=160",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_policy_not_created(self.policy_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=["setup", "-d", "*", self.cmd,
"-E", "setup.template.overwrite=true",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_policy_created(self.policy_name)
# check that template was overwritten
resp = self.es.transport.perform_request('GET', '/_index_template/' + self.custom_template)
found = False
for index_template in resp["index_templates"]:
if index_template["name"] == self.custom_template:
found = True
index = index_template["index_template"]["template"]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
assert found