Use gse instead gojieba

go.mod

@@ -49,7 +49,6 @@ require (
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/spf13/cast v1.4.1
 	github.com/spf13/viper v1.10.1
-	github.com/yanyiwu/gojieba v1.1.3-0.20220131084353-a30c1dcd29bb
 	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
 )
 
@@ -118,6 +117,7 @@ require (
 	github.com/tdewolff/parse/v2 v2.5.27 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect
+	github.com/vcaesar/cedar v0.20.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
@@ -150,6 +150,7 @@ require (
 require (
 	github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible // indirect
 	github.com/g4s8/hexcolor v1.1.0
+	github.com/go-ego/gse v0.70.0
 	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/iris-contrib/middleware/cors v0.0.0-20220218204641-8e282f24acb7
 	github.com/iris-contrib/middleware/jwt v0.0.0-20220301201128-27fa0f6a7d7e

go.sum

@@ -44,6 +44,8 @@ github.com/g4s8/hexcolor v1.1.0 h1:PGG9472oBEMH4SMM//BMvZcS47ThkWR9YNJnMPDMDXo=
 github.com/g4s8/hexcolor v1.1.0/go.mod h1:hG/pPv7k4KJmz8Zo/RQpawY6kwl7HraIBv12qOdrjo4=
 github.com/go-co-op/gocron v1.13.0 h1:BjkuNImPy5NuIPEifhWItFG7pYyr27cyjS6BN9w/D4c=
 github.com/go-co-op/gocron v1.13.0/go.mod h1:GD5EIEly1YNW+LovFVx5dzbYVcIc8544K99D8UVRpGo=
+github.com/go-ego/gse v0.70.0 h1:K9M+clnPc1sjnSGvLgSHAo39oWr0v3deXB/31K0n18w=
+github.com/go-ego/gse v0.70.0/go.mod h1:M9Xv8cEW7Of27BbE4p0iI3arqQHCYcm5N16/2b3pPPk=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
@@ -207,6 +209,9 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
+github.com/vcaesar/cedar v0.20.0 h1:VtBy/twzVjXiTo1Ij3fQRyDQRzvzDa9sKacpbwSJyps=
+github.com/vcaesar/cedar v0.20.0/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
+github.com/vcaesar/tt v0.20.0 h1:9t2Ycb9RNHcP0WgQgIaRKJBB+FrRdejuaL6uWIHuoBA=
 github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
 github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
@@ -219,8 +224,6 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
-github.com/yanyiwu/gojieba v1.1.3-0.20220131084353-a30c1dcd29bb h1:m5U8w4b/yUIucBKfxSKpUZVuTPSH+ZEg8yA5LxPeRGQ=
-github.com/yanyiwu/gojieba v1.1.3-0.20220131084353-a30c1dcd29bb/go.mod h1:54wkP7sMJ6bklf7yPl6F+JG71dzVUU1WigZbR47nGdY=
 github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA=
 github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0=
 github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=

modules/wordcloud/collector.go

@@ -1,23 +1,23 @@
 package wordcloud
 
 import (
-	"strings"
+	"github.com/go-ego/gse"
+)
 
-	"github.com/yanyiwu/gojieba"
+var (
+	seg gse.Segmenter
 )
 
 type WordCollector struct {
 	wordSet []WordJson
 }
 
 func NewWordCollectorWithStr(str string) *WordCollector {
-	wordAnalyzer := gojieba.NewJieba()
-	defer wordAnalyzer.Free()
-	words := wordAnalyzer.Tag(str)
-	roots, wordClass := getWordClass(words)
-	wordCounter := getWordCounter(roots)
-	return NewWordCollectorWithSet(generateWordSet(roots, wordClass, wordCounter))
-
+	words := seg.Pos(str, true)
+	words = seg.TrimPos(words)
+	words = seg.TrimWithPos(words, "x", "m", "eng")
+	wordSet := newWordSet(words)
+	return NewWordCollectorWithSet(wordSet)
 }
 
 func NewWordCollectorWithSet(wordSet []WordJson) *WordCollector {
@@ -26,30 +26,6 @@ func NewWordCollectorWithSet(wordSet []WordJson) *WordCollector {
 	}
 }
 
-func getWordClass(words []string) ([]string, []string) {
-	roots := make([]string, 0)
-	wordclass := make([]string, 0)
-	for _, word := range words {
-		spilt := strings.Split(word, "/")
-		roots = append(roots, spilt[0])
-		wordclass = append(wordclass, spilt[1])
-	}
-	return roots, wordclass
-}
-
-func getWordCounter(words []string) map[string]uint {
-	counter := make(map[string]uint, 0)
-	for _, word := range words {
-		_, ok := counter[word]
-		if !ok {
-			counter[word] = 1
-		} else {
-			counter[word]++
-		}
-	}
-	return counter
-}
-
 func accumulateWords(words []*WordJson) []*WordJson {
 	length := len(words)
 	ans := make([]*WordJson, 0)
@@ -75,24 +51,19 @@ func accumulateWords(words []*WordJson) []*WordJson {
 	return ans
 }
 
-func generateWordSet(roots []string, wordclass []string, wordCounter map[string]uint) []WordJson {
-	length := len(roots)
-	ans := make([]WordJson, 0)
-	uniqueMap := make(map[string]bool)
-	for i := 0; i < length; i++ {
-		if _, ok := wordCounter[roots[i]]; ok {
-			if ok := uniqueMap[roots[i]]; !ok {
-				uniqueMap[roots[i]] = true
-				ans = append(ans, WordJson{
-					Content:   roots[i],
-					WordClass: wordclass[i],
-					Count:     wordCounter[roots[i]],
-				})
-			}
-
-		}
+func newWordSet(words []gse.SegPos) (wordSet []WordJson) {
+	counter := map[gse.SegPos]uint{}
+	for _, word := range words {
+		counter[word]++
 	}
-	return ans
+	for word, count := range counter {
+		wordSet = append(wordSet, WordJson{
+			Content:   word.Text,
+			WordClass: word.Pos,
+			Count:     count,
+		})
+	}
+	return
 }
 
 func (wc *WordCollector) Filter(filter Filter) *WordCollector {

modules/wordcloud/index.go

@@ -29,6 +29,7 @@ var mctx *module.ModuleContext
 
 func entry(ctx *module.ModuleContext) {
 	mctx = ctx
+	seg.LoadDictEmbed()
 	mctx.Route.PartyFunc("/word", func(word iris.Party) {
 		word.Get("/", rbac.PermInterceptor("word.view"), getAllWords)
 		word.Get("/{id:uint}", rbac.PermInterceptor("word.view"), getWordsByOrder)

modules/wordcloud/listener.go

@@ -9,51 +9,65 @@ func listener() {
 		}
 	}()
 
-	orderModule := mctx.Registry.Get("order")
-	if orderModule == nil {
-		mctx.Logger.Errorf("order module not found")
-		return
-	}
-
 	for {
 		select {
 		// order created
 		case ch := <-mctx.EventBus.On("order:create"):
 			orderID, _ := ch.Args[0].(uint)
 			odr, err := order.GetOrderByID(orderID)
 			if err != nil {
-				mctx.Logger.Warnf("get order failed: %s", err)
+				mctx.Logger.Warnf("Get order failed: %s", err)
 				continue
 			}
-			uploadWordsService(odr.ID, odr.Title)
-			uploadWordsService(odr.ID, odr.Content)
+			if res := uploadWordsService(odr.ID, odr.Title); !res.Status {
+				mctx.Logger.Warnf("Upload words failed: [order: %d, content: %s] errors: %v", odr.ID, odr.Title, res.Data)
+			} else {
+				mctx.Logger.Infof("Upload words success: [order: %d, content: %s]", odr.ID, odr.Title)
+			}
+			if res := uploadWordsService(odr.ID, odr.Content); !res.Status {
+				mctx.Logger.Warnf("Upload words failed: [order: %d, content: %s] errors: %v", odr.ID, odr.Title, res.Data)
+			} else {
+				mctx.Logger.Infof("Upload words success: [order: %d, content: %s]", odr.ID, odr.Title)
+			}
 		// order title changed
 		case ch := <-mctx.EventBus.On("order:update:title"):
 			orderID, _ := ch.Args[0].(uint)
 			odr, err := order.GetOrderByID(orderID)
 			if err != nil {
-				mctx.Logger.Warnf("get order failed: %s", err)
+				mctx.Logger.Warnf("Get order failed: %s", err)
 				continue
 			}
-			uploadWordsService(odr.ID, odr.Title)
+			if res := uploadWordsService(odr.ID, odr.Title); !res.Status {
+				mctx.Logger.Warnf("Upload words failed: [order: %d, content: %s] errors: %v", odr.ID, odr.Title, res.Data)
+			} else {
+				mctx.Logger.Infof("Upload words success: [order: %d, content: %s]", odr.ID, odr.Title)
+			}
 		// order content changed
 		case ch := <-mctx.EventBus.On("order:update:content"):
 			orderID, _ := ch.Args[0].(uint)
 			odr, err := order.GetOrderByID(orderID)
 			if err != nil {
-				mctx.Logger.Warnf("get order failed: %s", err)
+				mctx.Logger.Warnf("Get order failed: %s", err)
 				continue
 			}
-			uploadWordsService(odr.ID, odr.Content)
+			if res := uploadWordsService(odr.ID, odr.Content); !res.Status {
+				mctx.Logger.Warnf("Upload words failed: [order: %d, content: %s] errors: %v", odr.ID, odr.Title, res.Data)
+			} else {
+				mctx.Logger.Infof("Upload words success: [order: %d, content: %s]", odr.ID, odr.Title)
+			}
 		// order comment
 		case ch := <-mctx.EventBus.On("order:update:comment"):
 			commentID, _ := ch.Args[1].(uint)
 			comment, err := order.GetCommentByID(commentID)
 			if err != nil {
-				mctx.Logger.Warnf("get comment failed: %s", err)
+				mctx.Logger.Warnf("Get comment failed: %s", err)
 				continue
 			}
-			uploadWordsService(comment.OrderID, comment.Content)
+			if res := uploadWordsService(comment.OrderID, comment.Content); !res.Status {
+				mctx.Logger.Warnf("Upload words failed: [order: %d, content: %s] errors: %v", comment.OrderID, comment.Content, res.Data)
+			} else {
+				mctx.Logger.Infof("Upload words success: [order: %d, content: %s]", comment.OrderID, comment.Content)
+			}
 		}
 	}
 }