💊 add cache for long text query; 🐛 fixed #1, #23, #24

cmd/kd.go

@@ -24,7 +24,7 @@ import (
 	"go.uber.org/zap"
 )
 
-var VERSION = "v0.0.6"
+var VERSION = "v0.0.7"
 
 func showPrompt() {
 	exename, err := pkg.GetExecutableBasename()
@@ -185,7 +185,6 @@ func flagStatus(*cli.Context, bool) error {
 	if err == nil {
 		fmt.Printf("    Binary地址：%s\n", kdpath)
 	}
-
 	return err
 }
 

internal/cache/cache.go

@@ -5,9 +5,11 @@ import (
 	"compress/zlib"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/Karmenzind/kd/internal/model"
 	"github.com/Karmenzind/kd/pkg"
@@ -76,6 +78,54 @@ func UpdateQueryCache(r *model.Result) (err error) {
 }
 
 //  -----------------------------------------------------------------------------
+//  Long Text query cache
+//  -----------------------------------------------------------------------------
+
+type LongTextData struct {
+	Result   string `json:"r"`
+	AccessTS int64  `json:"a"`
+	CreateTS int64  `json:"c"`
+}
+
+func GetLongTextCache(r *model.Result) (err error) {
+	var m map[string]LongTextData
+	if pkg.IsPathExists(LONG_TEXT_CACHE_FILE) {
+		err = pkg.LoadJson(LONG_TEXT_CACHE_FILE, &m)
+		if err != nil {
+			return err
+		}
+        if res, ok := m[r.Query]; ok {
+            r.MachineTrans = res.Result
+            zap.S().Debugf("Got cached '%s'", r.Query)
+            (&res).AccessTS = time.Now().Unix()
+            m[r.Query] = res
+            go pkg.SaveJson(LONG_TEXT_CACHE_FILE, &m)
+            return
+        } else {
+            return fmt.Errorf("no cache for %s", r.Query)
+        }
+    } 
+    return
+}
+
+func UpdateLongTextCache(r *model.Result) (err error) {
+	var m map[string]LongTextData
+	if pkg.IsPathExists(LONG_TEXT_CACHE_FILE) {
+		err = pkg.LoadJson(LONG_TEXT_CACHE_FILE, &m)
+		if err != nil {
+			return err
+		}
+	} else {
+		m = map[string]LongTextData{}
+	}
+	now := time.Now().Unix()
+	m[r.Query] = LongTextData{r.MachineTrans, now, now}
+	err = pkg.SaveJson(LONG_TEXT_CACHE_FILE, m)
+	return err
+}
+
+//  -----------------------------------------------------------------------------
+// deprecated
 //  JSON version
 //  -----------------------------------------------------------------------------
 

internal/cache/utils.go

@@ -3,6 +3,7 @@ package cache
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/Karmenzind/kd/internal/run"
 	d "github.com/Karmenzind/kd/pkg/decorate"
@@ -13,6 +14,8 @@ var CACHE_WORDS_PATH = run.CACHE_WORDS_PATH
 var CACHE_RUN_PATH = run.CACHE_RUN_PATH
 var CACHE_STAT_DIR_PATH = run.CACHE_STAT_DIR_PATH
 
+var LONG_TEXT_CACHE_FILE string
+
 func init() {
 	for _, directory := range []string{
 		CACHE_ROOT_PATH,
@@ -25,4 +28,5 @@ func init() {
 			d.EchoFatal(fmt.Sprintf("Failed to create %s", directory))
 		}
 	}
+    LONG_TEXT_CACHE_FILE = filepath.Join(CACHE_ROOT_PATH, "long_text_results.json")
 }

internal/client.go

@@ -66,8 +66,10 @@ func Query(query string, noCache bool, longText bool) (r *model.Result, err erro
 	daemonRunning := make(chan bool)
 	go ensureDaemon(daemonRunning)
 
-	core.WG.Add(1)
-	go cache.CounterIncr(query, r.History)
+	if !longText {
+		core.WG.Add(1)
+		go cache.CounterIncr(query, r.History)
+	}
 
 	// any valid char
 	if m, _ := regexp.MatchString("^[a-zA-Z0-9\u4e00-\u9fa5]", query); !m {
@@ -91,18 +93,12 @@ func Query(query string, noCache bool, longText bool) (r *model.Result, err erro
 			inNotFound = true
 		}
 		r.Initialize()
+	}
 
-		if !noCache {
-			cacheErr := q.FetchCached(r)
-			if cacheErr != nil {
-				zap.S().Warnf("[cache] Query error: %s", cacheErr)
-			}
-			if r.Found {
-				return
-			}
-			_ = err
+	if !noCache {
+		if cacheErr := q.FetchCached(r); cacheErr == nil && r.Found {
+			return
 		}
-
 	}
 
 	if <-daemonRunning {

internal/query/online.go

@@ -6,13 +6,15 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/Karmenzind/kd/config"
 	"github.com/Karmenzind/kd/internal/cache"
 	"github.com/Karmenzind/kd/internal/model"
+	"github.com/Karmenzind/kd/internal/run"
 	"github.com/Karmenzind/kd/pkg"
 	"github.com/anaskhan96/soup"
 	"go.uber.org/zap"
@@ -66,9 +68,16 @@ func requestYoudao(r *model.Result) (body []byte, err error) {
 		zap.S().Debugf("[http-get] detail: header %+v", url, len(body), resp.Header)
 	}
 	if config.Cfg.Debug {
-		errW := os.WriteFile(fmt.Sprintf("/home/k/Workspace/kd/data/%s.html", r.Query), (body), 0666)
-		if errW != nil {
-			zap.S().Warnf("Failed to write file '%s': %s", r.Query, errW)
+		htmlDir := filepath.Join(run.CACHE_ROOT_PATH, "html")
+		errW := os.MkdirAll(htmlDir, os.ModePerm)
+		if errW == nil {
+			htmlPath := filepath.Join(htmlDir, fmt.Sprintf("%s.html", r.Query))
+			errW = os.WriteFile(htmlPath, body, 0666)
+		}
+		if errW == nil {
+			zap.S().Debugf("Saved '%s.html'", r.Query)
+		} else {
+			zap.S().Warnf("Failed to save html data for '%s': %s", r.Query, errW)
 		}
 	}
 	return
@@ -102,6 +111,7 @@ func FetchOnline(r *model.Result) (err error) {
 		yr.parseMachineTrans()
 		if r.MachineTrans != "" {
 			r.Found = true
+            go cache.UpdateLongTextCache(r)
 		}
 		return
 	}

internal/query/output.go

@@ -150,7 +150,18 @@ func displayExample(item []string, tab string, onlyEN bool, isEN bool) string {
 		if onlyEN {
 			r = d.EgEn(item[0])
 		} else {
-			r = fmt.Sprintf("%s %s", d.EgEn(item[0]), d.EgCh(item[1]))
+			var rh string
+			if len(item) >= 3 {
+				// if strings.ToLower(item[2]) != "youdao" {
+				// 	rh = d.EgCh(item[1]) + d.EgCh(item[2])
+				// }
+					rh = d.EgCh(item[1]) + d.EgCh(item[2])
+			}
+			if rh == "" {
+				rh = d.EgCh(item[1])
+			}
+			r = fmt.Sprintf("%s %s", d.EgEn(item[0]), rh)
+			// r = fmt.Sprintf("%s %s", d.EgEn(item[0]), d.EgCh(item[1]))
 		}
 	case "au":
 		// TODO 增加来源渲染

internal/query/query.go

@@ -23,11 +23,16 @@ TODO:
 */
 
 func FetchCached(r *model.Result) (err error) {
-	err = cache.GetCachedQuery(r)
+    if r.IsLongText {
+        err = cache.GetLongTextCache(r)
+    } else {
+        err = cache.GetCachedQuery(r)
+    }
 	if err == nil {
 		r.Found = true
 		return
 	}
+    zap.S().Debugf("[cache] Query error: %s", err)
 	r.Found = false
 	return
 }

pkg/file.go

@@ -8,6 +8,7 @@ import (
 	"go.uber.org/zap"
 )
 
+
 func IsPathExists(p string) bool {
 	if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
 		return false

plan.md

@@ -5,8 +5,6 @@
 - 读文件加锁
 - 指定port
 - release增加version，aur判断此文件
-- 长句查询 (另外缓存)
-- notfound counter 忽略longtext
 - gitea镜像
 
 ## short-term
@@ -25,7 +23,6 @@
 - 加入词库设置，供选择词库大小
 
 ## low priority
-- 一键安装脚本
 - --update下载之后缓存，避免重复下载
 - cli替换为cobra
 - source数据 分为base-sourse & web-source
@@ -34,8 +31,8 @@
 - 检测配置保存时间变化的基础上再加上内容判断？
 - not found list记录查询时间，超时删除
 - not found和索引都加入服务端缓存，benchmark比较直接查本地和tcp通信的速度
-
 - AUR收尾工作
+- move default log to cache dir
 
 # BUG