Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add real tests for splunk logger using vector #9526

Merged
merged 10 commits into from
Jun 2, 2023
Merged

feat: Add real tests for splunk logger using vector #9526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
01b4b85
feat: add real tests for splunk logger
Revolyssup May 19, 2023
227c9c4
feat: Add real tests for splunk logger using vector
Revolyssup May 23, 2023
34c250e
readd lost tests
Revolyssup May 23, 2023
eb40d6e
fix failed auth test case
Revolyssup May 23, 2023
58af4fd
merge conf
Revolyssup May 26, 2023
4e64b0a
add newline
Revolyssup May 26, 2023
21920c2
fix
Revolyssup May 26, 2023
9ad0e14
add new line
Revolyssup May 29, 2023
d2d6b10
fix failing tests
Revolyssup May 30, 2023
71f8ce0
Merge remote-tracking branch 'upstream/master' into addSplunkTests
Revolyssup May 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions ci/pod/docker-compose.plugin.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,18 +196,6 @@ services:
networks:
opa_net:

# Splunk HEC Logging Service
splunk:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why remove this component?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because we don't need a separate splunk server now. Vector does the job

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

image: splunk/splunk:8.2.3
restart: unless-stopped
ports:
- "18088:8088"
environment:
SPLUNK_PASSWORD: "ApacheAPISIX@666"
SPLUNK_START_ARGS: "--accept-license"
SPLUNK_HEC_TOKEN: "BD274822-96AA-4DA6-90EC-18940FB2414C"
SPLUNK_HEC_SSL: "False"

# Elasticsearch Logger Service
elasticsearch-noauth:
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
Expand Down Expand Up @@ -332,6 +320,7 @@ services:
- '8127:8127/udp'
- '43000:43000'
- '5140:5140'
- "18088:18088" # For splunk logging tests
- '5150:5150/udp'
networks:
vector_net:
Expand Down
15 changes: 14 additions & 1 deletion ci/pod/vector/vector.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,15 @@ type = "syslog"
address = "0.0.0.0:5150"
mode = "udp"

[sources.log-from-splunk]
type = "splunk_hec"
address = "0.0.0.0:18088"
valid_tokens = [
"BD274822-96AA-4DA6-90EC-18940FB2414C"
]

[sinks.log-2-console]
inputs = [ "log-from-tcp", "log-from-tls", "log-from-syslog-tcp", "log-from-syslog-udp", "log-from-udp"]
inputs = [ "log-from-tcp", "log-from-tls", "log-from-syslog-tcp", "log-from-syslog-udp", "log-from-udp", "log-from-splunk"]
type = "console"
encoding.codec = "json"

Expand Down Expand Up @@ -81,6 +88,12 @@ type = "file"
encoding.codec = "text"
path = "/etc/vector/syslog-tcp.log"

[sinks.log-2-splunk-file]
inputs = [ "log-from-splunk" ]
type = "file"
encoding.codec = "json"
path = "/etc/vector/splunk.log"

[sinks.log-2-syslog-udp-file]
inputs = [ "log-from-syslog-udp" ]
type = "file"
Expand Down
190 changes: 97 additions & 93 deletions t/plugin/splunk-hec-logging.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ GET /hello
--- response_body
hello world
--- error_log
Batch Processor[splunk-hec-logging] failed to process entries: failed to send splunk, Invalid token
Batch Processor[splunk-hec-logging] failed to process entries: failed to send splunk, Invalid authorization
Batch Processor[splunk-hec-logging] exceeded the max_retry_count


Expand Down Expand Up @@ -224,157 +224,139 @@ hello world
--- config
location /t {
content_by_lua_block {
local config = {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr",
"message_1":"test custom log format in plugin"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end

local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
["splunk-hec-logging"] = {
endpoint = {
uri = "http://127.0.0.1:1980/splunk_hec_logging",
uri = "http://127.0.0.1:18088/services/collector",
token = "BD274822-96AA-4DA6-90EC-18940FB2414C"
},
batch_max_size = 1,
batch_max_size = 3,
inactive_timeout = 1
}
}
}
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, config)

})
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end

local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr"
}
}]]
)
local code, _, body2 = t("/hello", "GET")
if code >= 300 then
ngx.status = code
ngx.say(body)
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5



=== TEST 8: hit
--- extra_init_by_lua
local core = require("apisix.core")
local decode = require("toolkit.json").decode
local up = require("lib.server")
up.splunk_hec_logging = function()
ngx.log(ngx.WARN, "the mock backend is hit")

ngx.req.read_body()
local data = ngx.req.get_body_data()
ngx.log(ngx.WARN, data)
data = decode(data)
assert(data.event.client_ip == "127.0.0.1")
assert(data.source == "apache-apisix-splunk-hec-logging")
assert(data.host == core.utils.gethostname())
ngx.say('{}')
end
--- request
GET /hello
--- wait: 2
--- response_body
hello world
--- error_log
the mock backend is hit
--- no_error_log
[error]
=== TEST 8: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
qr/.*test custom log format in plugin.*/



=== TEST 9: set route to test custom log format in route
--- config
location /t {
content_by_lua_block {
local config = {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"vip": "$remote_addr",
"message_2":"logger format in plugin"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end

local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need a blank

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added newline at all suggested place @monkeyDluffy6017

uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
["splunk-hec-logging"] = {
endpoint = {
uri = "http://127.0.0.1:1980/splunk_hec_logging",
uri = "http://127.0.0.1:18088/services/collector",
token = "BD274822-96AA-4DA6-90EC-18940FB2414C"
},
log_format = {
host = "$host",
["@timestamp"] = "$time_iso8601",
vip = "$remote_addr"
},
batch_max_size = 1,
batch_max_size = 3,
inactive_timeout = 1
}
}
}
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, config)

})
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end

local code, _, body2 = t("/hello", "GET")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need a blank

if code >= 300 then
ngx.status = code
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5



=== TEST 10: hit
--- extra_init_by_lua
local core = require("apisix.core")
local decode = require("toolkit.json").decode
local up = require("lib.server")
up.splunk_hec_logging = function()
ngx.log(ngx.WARN, "the mock backend is hit")

ngx.req.read_body()
local data = ngx.req.get_body_data()
ngx.log(ngx.WARN, data)
data = decode(data)
assert(data.event.vip == "127.0.0.1")
assert(data.source == "apache-apisix-splunk-hec-logging")
assert(data.host == core.utils.gethostname())
ngx.say('{}')
end
--- request
GET /hello
--- wait: 2
--- response_body
hello world
--- error_log
the mock backend is hit
--- no_error_log
[error]
=== TEST 10: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
qr/.*logger format in plugin.*/



Expand All @@ -383,12 +365,29 @@ the mock backend is hit
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/plugin_metadata/splunk-hec-logging',
ngx.HTTP_PUT,
[[{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"vip": "$remote_addr",
"message_3":"test batched data"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need a blank


local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, {
uri = "/hello",
upstream = {
type = "roundrobin",
nodes = {
["127.0.0.1:1980"] = 1
["127.0.0.1:1982"] = 1
}
},
plugins = {
Expand All @@ -406,19 +405,24 @@ the mock backend is hit
if code >= 300 then
ngx.status = code
end

local code, _, body2 = t("/hello", "GET")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need a blank

if code >= 300 then
ngx.status = code
ngx.say("fail")
return
end
ngx.say(body)
}
}
--- response_body
passed
--- wait: 5



=== TEST 12: hit
--- pipelined_requests eval
["GET /hello", "GET /hello", "GET /hello"]
--- wait: 2
=== TEST 12: check splunk log
--- exec
tail -n 1 ci/pod/vector/splunk.log
--- response_body eval
["hello world\n", "hello world\n", "hello world\n"]
--- no_error_log
[error]
qr/.*test batched data.*/