optimize: enhance log by elasticsearch and kibana (#118)

* remove: ants pkg

* feat: enhance log by elasticsearch and kibana

cmd/api/biz/middleware/es/init.go

@@ -0,0 +1,39 @@
+package es
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/ozline/tiktok/config"
+	"github.com/ozline/tiktok/pkg/constants"
+	"github.com/ozline/tiktok/pkg/eslogrus"
+	"github.com/sirupsen/logrus"
+
+	elasticsearch "github.com/elastic/go-elasticsearch"
+)
+
+var (
+	EsClient *elasticsearch.Client
+)
+
+func EsHookLog() *eslogrus.ElasticHook {
+	hook, err := eslogrus.NewElasticHook(EsClient, config.Elasticsearch.Host, logrus.DebugLevel, constants.APIServiceName)
+	if err != nil {
+		panic(err)
+	}
+
+	return hook
+}
+
+// InitEs 初始化es
+func Init() {
+	esConn := fmt.Sprintf("http://%s", config.Elasticsearch.Addr)
+	cfg := elasticsearch.Config{
+		Addresses: []string{esConn},
+	}
+	client, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		log.Panic(err)
+	}
+	EsClient = client
+}

cmd/chat/dal/cache/init.go

@@ -5,14 +5,12 @@ import (
 	"time"
 
 	"github.com/ozline/tiktok/config"
-	"github.com/panjf2000/ants/v2"
 	redis "github.com/redis/go-redis/v9"
 	"gorm.io/gorm"
 )
 
 var (
-	RedisDB  *redis.Client
-	AntsPool *ants.PoolWithFunc
+	RedisDB *redis.Client
 )
 
 type Message struct {

cmd/chat/dal/init.go

@@ -4,7 +4,6 @@ import (
 	"github.com/ozline/tiktok/cmd/chat/dal/cache"
 	"github.com/ozline/tiktok/cmd/chat/dal/db"
 	"github.com/ozline/tiktok/cmd/chat/dal/mq"
-	"github.com/ozline/tiktok/pkg/ants"
 )
 
 func Init() {
@@ -13,5 +12,4 @@ func Init() {
 	mq.InitRabbitMQ()
 	mq.InitMessageMQ()
 	mq.InitChatMQ()
-	ants.Init()
 }

cmd/chat/main.go

@@ -2,27 +2,33 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net"
 
 	"github.com/cloudwego/kitex/pkg/klog"
 	"github.com/cloudwego/kitex/pkg/limit"
 	"github.com/cloudwego/kitex/pkg/rpcinfo"
 	"github.com/cloudwego/kitex/server"
+	"github.com/elastic/go-elasticsearch"
+	kitexlogrus "github.com/kitex-contrib/obs-opentelemetry/logging/logrus"
 	etcd "github.com/kitex-contrib/registry-etcd"
+	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/ozline/tiktok/cmd/api/biz/middleware/es"
+	"github.com/ozline/tiktok/cmd/chat/dal"
 	"github.com/ozline/tiktok/config"
 	chat "github.com/ozline/tiktok/kitex_gen/chat/messageservice"
-
-	"github.com/ozline/tiktok/cmd/chat/dal"
 	"github.com/ozline/tiktok/pkg/constants"
+	"github.com/ozline/tiktok/pkg/eslogrus"
 	"github.com/ozline/tiktok/pkg/tracer"
 	"github.com/ozline/tiktok/pkg/utils"
-
-	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/sirupsen/logrus"
 )
 
 var (
 	path       *string
 	listenAddr string // listen port
+
+	EsClient *elasticsearch.Client
 )
 
 func Init() {
@@ -33,6 +39,32 @@ func Init() {
 
 	dal.Init()
 	tracer.InitJaeger(constants.ChatServiceName)
+
+	EsInit()
+	klog.SetLevel(klog.LevelDebug)
+	klog.SetLogger(kitexlogrus.NewLogger(kitexlogrus.WithHook(es.EsHookLog())))
+}
+
+func EsHookLog() *eslogrus.ElasticHook {
+	hook, err := eslogrus.NewElasticHook(EsClient, config.Elasticsearch.Host, logrus.DebugLevel, constants.ChatServiceName)
+	if err != nil {
+		panic(err)
+	}
+
+	return hook
+}
+
+// InitEs 初始化es
+func EsInit() {
+	esConn := fmt.Sprintf("http://%s", config.Elasticsearch.Addr)
+	cfg := elasticsearch.Config{
+		Addresses: []string{esConn},
+	}
+	client, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		panic(err)
+	}
+	EsClient = client
 }
 
 func main() {

cmd/follow/main.go

@@ -2,26 +2,34 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net"
 
 	"github.com/cloudwego/kitex/pkg/klog"
 	"github.com/cloudwego/kitex/pkg/limit"
 	"github.com/cloudwego/kitex/pkg/rpcinfo"
 	"github.com/cloudwego/kitex/server"
+	"github.com/elastic/go-elasticsearch"
+	kitexlogrus "github.com/kitex-contrib/obs-opentelemetry/logging/logrus"
 	etcd "github.com/kitex-contrib/registry-etcd"
 	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/ozline/tiktok/cmd/api/biz/middleware/es"
 	"github.com/ozline/tiktok/cmd/follow/dal"
 	"github.com/ozline/tiktok/cmd/follow/rpc"
 	"github.com/ozline/tiktok/config"
 	follow "github.com/ozline/tiktok/kitex_gen/follow/followservice"
 	"github.com/ozline/tiktok/pkg/constants"
+	"github.com/ozline/tiktok/pkg/eslogrus"
 	"github.com/ozline/tiktok/pkg/tracer"
 	"github.com/ozline/tiktok/pkg/utils"
+	"github.com/sirupsen/logrus"
 )
 
 var (
 	path       *string
 	listenAddr string // listen port
+
+	EsClient *elasticsearch.Client
 )
 
 func Init() {
@@ -33,6 +41,32 @@ func Init() {
 	dal.Init()
 	rpc.Init()
 	tracer.InitJaeger(constants.FollowServiceName)
+
+	EsInit()
+	klog.SetLevel(klog.LevelDebug)
+	klog.SetLogger(kitexlogrus.NewLogger(kitexlogrus.WithHook(es.EsHookLog())))
+}
+
+func EsHookLog() *eslogrus.ElasticHook {
+	hook, err := eslogrus.NewElasticHook(EsClient, config.Elasticsearch.Host, logrus.DebugLevel, constants.FollowServiceName)
+	if err != nil {
+		panic(err)
+	}
+
+	return hook
+}
+
+// InitEs 初始化es
+func EsInit() {
+	esConn := fmt.Sprintf("http://%s", config.Elasticsearch.Addr)
+	cfg := elasticsearch.Config{
+		Addresses: []string{esConn},
+	}
+	client, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		panic(err)
+	}
+	EsClient = client
 }
 
 func main() {

cmd/interaction/main.go

@@ -2,26 +2,34 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net"
 
 	"github.com/cloudwego/kitex/pkg/klog"
 	"github.com/cloudwego/kitex/pkg/limit"
 	"github.com/cloudwego/kitex/pkg/rpcinfo"
 	"github.com/cloudwego/kitex/server"
+	"github.com/elastic/go-elasticsearch"
+	kitexlogrus "github.com/kitex-contrib/obs-opentelemetry/logging/logrus"
 	etcd "github.com/kitex-contrib/registry-etcd"
 	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/ozline/tiktok/cmd/api/biz/middleware/es"
 	"github.com/ozline/tiktok/cmd/interaction/dal"
 	"github.com/ozline/tiktok/cmd/interaction/rpc"
 	"github.com/ozline/tiktok/config"
 	interaction "github.com/ozline/tiktok/kitex_gen/interaction/interactionservice"
 	"github.com/ozline/tiktok/pkg/constants"
+	"github.com/ozline/tiktok/pkg/eslogrus"
 	"github.com/ozline/tiktok/pkg/tracer"
 	"github.com/ozline/tiktok/pkg/utils"
+	"github.com/sirupsen/logrus"
 )
 
 var (
 	path       *string
 	listenAddr string // listen port
+
+	EsClient *elasticsearch.Client
 )
 
 func Init() {
@@ -33,6 +41,32 @@ func Init() {
 	rpc.Init()
 	dal.Init()
 	tracer.InitJaeger(constants.InteractionServiceName)
+
+	EsInit()
+	klog.SetLevel(klog.LevelDebug)
+	klog.SetLogger(kitexlogrus.NewLogger(kitexlogrus.WithHook(es.EsHookLog())))
+}
+
+func EsHookLog() *eslogrus.ElasticHook {
+	hook, err := eslogrus.NewElasticHook(EsClient, config.Elasticsearch.Host, logrus.DebugLevel, constants.InteractionServiceName)
+	if err != nil {
+		panic(err)
+	}
+
+	return hook
+}
+
+// InitEs 初始化es
+func EsInit() {
+	esConn := fmt.Sprintf("http://%s", config.Elasticsearch.Addr)
+	cfg := elasticsearch.Config{
+		Addresses: []string{esConn},
+	}
+	client, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		panic(err)
+	}
+	EsClient = client
 }
 
 func main() {

cmd/user/main.go

@@ -2,26 +2,33 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net"
 
 	"github.com/cloudwego/kitex/pkg/klog"
 	"github.com/cloudwego/kitex/pkg/limit"
 	"github.com/cloudwego/kitex/pkg/rpcinfo"
 	"github.com/cloudwego/kitex/server"
+	"github.com/elastic/go-elasticsearch"
+	kitexlogrus "github.com/kitex-contrib/obs-opentelemetry/logging/logrus"
 	etcd "github.com/kitex-contrib/registry-etcd"
+	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/ozline/tiktok/cmd/api/biz/middleware/es"
 	"github.com/ozline/tiktok/cmd/user/dal"
 	"github.com/ozline/tiktok/config"
 	user "github.com/ozline/tiktok/kitex_gen/user/userservice"
 	"github.com/ozline/tiktok/pkg/constants"
+	"github.com/ozline/tiktok/pkg/eslogrus"
 	"github.com/ozline/tiktok/pkg/tracer"
 	"github.com/ozline/tiktok/pkg/utils"
-
-	trace "github.com/kitex-contrib/tracer-opentracing"
+	"github.com/sirupsen/logrus"
 )
 
 var (
 	path       *string
 	listenAddr string // listen port
+
+	EsClient *elasticsearch.Client
 )
 
 func Init() {
@@ -33,6 +40,32 @@ func Init() {
 	// others
 	dal.Init()
 	tracer.InitJaeger(constants.UserServiceName)
+
+	EsInit()
+	klog.SetLevel(klog.LevelDebug)
+	klog.SetLogger(kitexlogrus.NewLogger(kitexlogrus.WithHook(es.EsHookLog())))
+}
+
+func EsHookLog() *eslogrus.ElasticHook {
+	hook, err := eslogrus.NewElasticHook(EsClient, config.Elasticsearch.Host, logrus.DebugLevel, constants.FollowServiceName)
+	if err != nil {
+		panic(err)
+	}
+
+	return hook
+}
+
+// InitEs 初始化es
+func EsInit() {
+	esConn := fmt.Sprintf("http://%s", config.Elasticsearch.Addr)
+	cfg := elasticsearch.Config{
+		Addresses: []string{esConn},
+	}
+	client, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		panic(err)
+	}
+	EsClient = client
 }
 
 func main() {