Skip to content

Commit

Permalink
feat: Add real tests for splunk logger using vector (#9526)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Revolyssup
Revolyssup authored Jun 2, 2023
1 parent d058e45 commit 6ba4f73
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 106 deletions.
13 changes: 1 addition & 12 deletions ci/pod/docker-compose.plugin.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,18 +196,6 @@ services:
networks:
opa_net:

# Splunk HEC Logging Service
splunk:
image: splunk/splunk:8.2.3
restart: unless-stopped
ports:
- "18088:8088"
environment:
SPLUNK_PASSWORD: "ApacheAPISIX@666"
SPLUNK_START_ARGS: "--accept-license"
SPLUNK_HEC_TOKEN: "BD274822-96AA-4DA6-90EC-18940FB2414C"
SPLUNK_HEC_SSL: "False"

# Elasticsearch Logger Service
elasticsearch-noauth:
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
Expand Down Expand Up @@ -332,6 +320,7 @@ services:
- '8127:8127/udp'
- '43000:43000'
- '5140:5140'
- "18088:18088" # For splunk logging tests
- '5150:5150/udp'
networks:
vector_net:
Expand Down
15 changes: 14 additions & 1 deletion ci/pod/vector/vector.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,15 @@ type = "syslog"
address = "0.0.0.0:5150"
mode = "udp"

[sources.log-from-splunk]
type = "splunk_hec"
address = "0.0.0.0:18088"
valid_tokens = [
"BD274822-96AA-4DA6-90EC-18940FB2414C"
]

[sinks.log-2-console]
inputs = [ "log-from-tcp", "log-from-tls", "log-from-syslog-tcp", "log-from-syslog-udp", "log-from-udp"]
inputs = [ "log-from-tcp", "log-from-tls", "log-from-syslog-tcp", "log-from-syslog-udp", "log-from-udp", "log-from-splunk"]
type = "console"
encoding.codec = "json"

Expand Down Expand Up @@ -81,6 +88,12 @@ type = "file"
encoding.codec = "text"
path = "/etc/vector/syslog-tcp.log"

[sinks.log-2-splunk-file]
inputs = [ "log-from-splunk" ]
type = "file"
encoding.codec = "json"
path = "/etc/vector/splunk.log"

[sinks.log-2-syslog-udp-file]
inputs = [ "log-from-syslog-udp" ]
type = "file"
Expand Down
190 changes: 97 additions & 93 deletions t/plugin/splunk-hec-logging.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ GET /hello
--- response_body
hello world
--- error_log
Batch Processor[splunk-hec-logging] failed to process entries: failed to send splunk, Invalid token
Batch Processor[splunk-hec-logging] failed to process entries: failed to send splunk, Invalid authorization
Batch Processor[splunk-hec-logging] exceeded the max_retry_count
Expand Down Expand Up @@ -224,157 +224,139 @@ hello world
--- config
location /t {
content_by_lua_block {
local config = {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr",
"message_1":"test custom log format in plugin"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
["splunk-hec-logging"] = {
endpoint = {
uri = "http://127.0.0.1:1980/splunk_hec_logging",
uri = "http://127.0.0.1:18088/services/collector",
token = "BD274822-96AA-4DA6-90EC-18940FB2414C"
},
batch_max_size = 1,
batch_max_size = 3,
inactive_timeout = 1
}
}
}
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, config)
})
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr"
}
}]]
)
local code, _, body2 = t("/hello", "GET")
if code >= 300 then
ngx.status = code
ngx.say(body)
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5
=== TEST 8: hit
--- extra_init_by_lua
local core = require("apisix.core")
local decode = require("toolkit.json").decode
local up = require("lib.server")
up.splunk_hec_logging = function()
ngx.log(ngx.WARN, "the mock backend is hit")
ngx.req.read_body()
local data = ngx.req.get_body_data()
ngx.log(ngx.WARN, data)
data = decode(data)
assert(data.event.client_ip == "127.0.0.1")
assert(data.source == "apache-apisix-splunk-hec-logging")
assert(data.host == core.utils.gethostname())
ngx.say('{}')
end
--- request
GET /hello
--- wait: 2
--- response_body
hello world
--- error_log
the mock backend is hit
--- no_error_log
[error]
=== TEST 8: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
qr/.*test custom log format in plugin.*/
=== TEST 9: set route to test custom log format in route
--- config
location /t {
content_by_lua_block {
local config = {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"vip": "$remote_addr",
"message_2":"logger format in plugin"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
["splunk-hec-logging"] = {
endpoint = {
uri = "http://127.0.0.1:1980/splunk_hec_logging",
uri = "http://127.0.0.1:18088/services/collector",
token = "BD274822-96AA-4DA6-90EC-18940FB2414C"
},
log_format = {
host = "$host",
["@timestamp"] = "$time_iso8601",
vip = "$remote_addr"
},
batch_max_size = 1,
batch_max_size = 3,
inactive_timeout = 1
}
}
}
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, config)
})
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local code, _, body2 = t("/hello", "GET")
if code >= 300 then
ngx.status = code
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5
=== TEST 10: hit
--- extra_init_by_lua
local core = require("apisix.core")
local decode = require("toolkit.json").decode
local up = require("lib.server")
up.splunk_hec_logging = function()
ngx.log(ngx.WARN, "the mock backend is hit")
ngx.req.read_body()
local data = ngx.req.get_body_data()
ngx.log(ngx.WARN, data)
data = decode(data)
assert(data.event.vip == "127.0.0.1")
assert(data.source == "apache-apisix-splunk-hec-logging")
assert(data.host == core.utils.gethostname())
ngx.say('{}')
end
--- request
GET /hello
--- wait: 2
--- response_body
hello world
--- error_log
the mock backend is hit
--- no_error_log
[error]
=== TEST 10: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
qr/.*logger format in plugin.*/
Expand All @@ -383,12 +365,29 @@ the mock backend is hit
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"vip": "$remote_addr",
"message_3":"test batched data"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
Expand All @@ -406,19 +405,24 @@ the mock backend is hit
if code >= 300 then
ngx.status = code
end
local code, _, body2 = t("/hello", "GET")
if code >= 300 then
ngx.status = code
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5
=== TEST 12: hit
--- pipelined_requests eval
["GET /hello", "GET /hello", "GET /hello"]
--- wait: 2
=== TEST 12: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
["hello world\n", "hello world\n", "hello world\n"]
--- no_error_log
[error]
qr/.*test batched data.*/

0 comments on commit 6ba4f73

Please sign in to comment.