feat(sls-logger): support custom log format (apache#7328)

Fix apache#7129
Signed-off-by: spacewander <spacewanderlzx@gmail.com>

apisix/plugins/sls-logger.lua

@@ -17,6 +17,9 @@
 local core = require("apisix.core")
 local log_util = require("apisix.utils.log-util")
 local bp_manager_mod = require("apisix.utils.batch-processor-manager")
+local plugin = require("apisix.plugin")
+
+
 local plugin_name = "sls-logger"
 local ngx = ngx
 local rf5424 = require("apisix.plugins.slslog.rfc5424")
@@ -127,10 +130,15 @@ end
 
 -- log phase in APISIX
 function _M.log(conf, ctx)
-    local entry = log_util.get_full_log(ngx, conf)
-    if not entry.route_id then
-        core.log.error("failed to obtain the route id for sys logger")
-        return
+    local metadata = plugin.plugin_metadata(plugin_name)
+    local entry
+
+    if metadata and metadata.value.log_format
+       and core.table.nkeys(metadata.value.log_format) > 0
+    then
+        entry = log_util.get_custom_format_log(ctx, metadata.value.log_format)
+    else
+        entry = log_util.get_full_log(ngx, conf)
     end
 
     local json_str, err = core.json.encode(entry)

docs/en/latest/plugins/sls-logger.md

@@ -49,6 +49,40 @@ It might take some time to receive the log data. It will be automatically sent a
 
 This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.
 
+## Metadata
+
+You can also set the format of the logs by configuring the Plugin metadata. The following configurations are available:
+
+| Name       | Type   | Required | Default                                                                       | Description                                                                                                                                                                                                                                             |
+| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| log_format | object | False    | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
+
+:::info IMPORTANT
+
+Configuring the Plugin metadata is global in scope. This means that it will take effect on all Routes and Services which use the `sls-logger` Plugin.
+
+:::
+
+The example below shows how you can configure through the Admin API:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/sls-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "log_format": {
+        "host": "$host",
+        "@timestamp": "$time_iso8601",
+        "client_ip": "$remote_addr"
+    }
+}'
+```
+
+With this configuration, your logs would be formatted as shown below:
+
+```shell
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+```
+
 ## Enabling the Plugin
 
 The example below shows how you can configure the Plugin on a specific Route:

docs/zh/latest/plugins/sls-logger.md

@@ -46,6 +46,32 @@ title: sls-logger
 
 本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置) 配置部分。
 
+## 插件元数据设置
+
+| 名称             | 类型    | 必选项 | 默认值        | 有效值  | 描述                                             |
+| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
+| log_format       | object  | 可选   | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} |         | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../../../en/latest/apisix-variable.md) 或 [Nginx 内置变量](http://nginx.org/en/docs/varindex.html)。特别的，**该设置是全局生效的**，意味着指定 log_format 后，将对所有绑定 sls-logger 的 Route 或 Service 生效。 |
+
+### 设置日志格式示例
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/sls-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "log_format": {
+        "host": "$host",
+        "@timestamp": "$time_iso8601",
+        "client_ip": "$remote_addr"
+    }
+}'
+```
+
+在日志收集处，将得到类似下面的日志：
+
+```shell
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+```
+
 ## 如何开启
 
 1. 下面例子展示了如何为指定路由开启 `sls-logger` 插件的。

t/plugin/sls-logger.t

@@ -198,3 +198,49 @@ hello world
 --- response_body
 passed
 --- timeout: 5
+
+
+
+=== TEST 8: add log format
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/sls-logger',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "log_format": {
+                        "host": "$host",
+                        "client_ip": "$remote_addr"
+                    }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 9: access
+--- extra_init_by_lua
+    local json = require("toolkit.json")
+    local rfc5424 = require("apisix.plugins.slslog.rfc5424")
+    local old_f = rfc5424.encode
+    rfc5424.encode = function(facility, severity, hostname, appname, pid, project,
+                   logstore, access_key_id, access_key_secret, msg)
+        local r = json.decode(msg)
+        assert(r.client_ip == "127.0.0.1", r.client_ip)
+        assert(r.host == "localhost", r.host)
+        return old_f(facility, severity, hostname, appname, pid, project,
+                     logstore, access_key_id, access_key_secret, msg)
+    end
+--- request
+GET /hello
+--- response_body
+hello world