feat: add sensitive algorithm

config/config.go

@@ -34,6 +34,7 @@ var Config struct {
 	AdminOnly          bool     `env:"ADMIN_ONLY" envDefault:"false"`
 	HolePurgeDivisions []int    `env:"HOLE_PURGE_DIVISIONS" envDefault:"2"`
 	HolePurgeDays      int      `env:"HOLE_PURGE_DAYS" envDefault:"30"`
+	OpenSensitiveCheck bool     `env:"OPEN_SENSITIVE_CHECK" envDefault:"true"`
 }
 
 var DynamicConfig struct {

data/data.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/goccy/go-json"
+	"github.com/importcjj/sensitive"
 	"github.com/rs/zerolog/log"
 )
 
@@ -16,7 +17,7 @@ var MetaFile []byte
 
 var NamesMapping map[string]string
 
-var SensitiveWords []string
+var SensitiveWordFilter *sensitive.Filter
 
 func init() {
 	err := initNamesMapping()
@@ -40,10 +41,11 @@ func initNamesMapping() error {
 }
 
 func initSensitiveWords() error {
-	sensitiveWordsData, err := os.ReadFile(`data/sensitive_words.json`)
+	SensitiveWordFilter = sensitive.New()
+	err := SensitiveWordFilter.LoadWordDict("data/sensitive_words.txt")
 	if err != nil {
+		SensitiveWordFilter = nil
 		return err
 	}
-
-	return json.Unmarshal(sensitiveWordsData, &SensitiveWords)
+	return nil
 }

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

go.sum

@@ -90,6 +90,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hetiansu5/urlquery v1.2.7 h1:jn0h+9pIRqUziSPnRdK/gJK8S5TCnk+HZZx5fRHf8K0=
 github.com/hetiansu5/urlquery v1.2.7/go.mod h1:wFpZdTHRdwt7mk0EM/DdZEWtEN4xf8HJoH/BLXm/PG0=
+github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b h1:9hudrgWUhyfR4FRMOfL9KB1uYw48DUdHkkgr9ODOw7Y=
+github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b/go.mod h1:zLVdX6Ed2SvCbEamKmve16U0E03UkdJo4ls1TBfmc8Q=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=

utils/sensitive.go

@@ -1,5 +1,18 @@
 package utils
 
+import (
+	"treehole_next/config"
+	"treehole_next/data"
+)
+
 func IsSensitive(content string) bool {
+	if !config.Config.OpenSensitiveCheck {
+		return false
+	}
+
+	if data.SensitiveWordFilter != nil {
+		in, _ := data.SensitiveWordFilter.FindIn(content)
+		return in
+	}
 	return true
 }

utils/sensitive_test.go

@@ -0,0 +1,41 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/importcjj/sensitive"
+	"github.com/stretchr/testify/assert"
+
+	"treehole_next/config"
+	"treehole_next/data"
+)
+
+func TestIsSensitive(t *testing.T) {
+	data.SensitiveWordFilter = sensitive.New()
+	data.SensitiveWordFilter.AddWord("天气", "小明", "123")
+	config.Config.OpenSensitiveCheck = true
+	type args struct {
+		content string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "test1",
+			args: args{content: "今天天气真好"},
+			want: true,
+		},
+		{
+			name: "test2",
+			args: args{content: "昨天吃了什么"},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, IsSensitive(tt.args.content), "IsSensitive(%v)", tt.args.content)
+		})
+	}
+}