Skip to content

Commit

Permalink
Merge pull request #55 from basenana/fix/create
Browse files Browse the repository at this point in the history 
make post meilisearch sync
  • Loading branch information
@zwwhdls
zwwhdls authored Dec 14, 2024
2 parents ae38135 + 6a35b16 commit c98e5eb
Show file tree
Hide file tree
Showing 5 changed files with 159 additions and 98 deletions.
153 changes: 83 additions & 70 deletions api/doc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package api
import (
"fmt"
"strconv"
"strings"

"github.com/gin-gonic/gin"

Expand All @@ -44,6 +45,9 @@ func (s *HttpServer) store() gin.HandlerFunc {
// store the document
doc := body.ToDocument()
if err := s.chain.Store(c, doc); err != nil {
if strings.Contains(err.Error(), "already exists") {
return
}
c.String(500, fmt.Sprintf("store document error: %s", err))
return
}
Expand Down Expand Up @@ -123,85 +127,22 @@ func (s *HttpServer) get() gin.HandlerFunc {

func (s *HttpServer) filter() gin.HandlerFunc {
return func(c *gin.Context) {
namespace := c.Param("namespace")
page, err := strconv.Atoi(c.DefaultQuery("page", "0"))
if err != nil {
c.String(400, fmt.Sprintf("invalid page number: %s", c.Query("page")))
return
}
pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
if err != nil {
c.String(400, fmt.Sprintf("invalid pagesize: %s", c.Query("page")))
docQuery := getFilterQuery(c)
if docQuery == nil {
return
}
docQuery := DocQuery{
Namespace: namespace,
Source: c.Query("source"),
WebUrl: c.Query("webUrl"),
ParentID: c.Query("parentID"),
Search: c.Query("search"),
HitsPerPage: int64(pageSize),
Page: int64(page),
Sort: c.DefaultQuery("sort", "createdAt"),
Desc: c.DefaultQuery("desc", "false") == "true",
}
createAtStart := c.Query("createAtStart")
if createAtStart != "" {
createAtStartTimestamp, err := strconv.Atoi(createAtStart)
if err != nil {
c.String(400, fmt.Sprintf("invalid createAtStart: %s", c.Query("page")))
return
}
docQuery.CreatedAtStart = utils.ToPtr(int64(createAtStartTimestamp))
}
createAtEnd := c.Query("createAtEnd")
if createAtEnd != "" {
createAtEndTimestamp, err := strconv.Atoi(createAtEnd)
if err != nil {
c.String(400, fmt.Sprintf("invalid createAtEnd: %s", c.Query("page")))
return
}
docQuery.ChangedAtEnd = utils.ToPtr(int64(createAtEndTimestamp))
}
updatedAtStart := c.Query("updatedAtStart")
if updatedAtStart != "" {
updatedAtStartTimestamp, err := strconv.Atoi(updatedAtStart)
if err != nil {
c.String(400, fmt.Sprintf("invalid updatedAtStart: %s", c.Query("page")))
return
}
docQuery.ChangedAtStart = utils.ToPtr(int64(updatedAtStartTimestamp))
}
updatedAtEnd := c.Query("updatedAtEnd")
if updatedAtEnd != "" {
updatedAtEndTimestamp, err := strconv.Atoi(updatedAtEnd)
if err != nil {
c.String(400, fmt.Sprintf("invalid updatedAtEnd: %s", c.Query("page")))
return
}
docQuery.ChangedAtEnd = utils.ToPtr(int64(updatedAtEndTimestamp))
}
fuzzyName := c.Query("fuzzyName")
if fuzzyName != "" {
docQuery.FuzzyName = &fuzzyName
}
if c.Query("unRead") != "" {
docQuery.UnRead = utils.ToPtr(c.Query("unRead") == "true")
}
if c.Query("mark") != "" {
docQuery.Mark = utils.ToPtr(c.Query("mark") == "true")
}
docs, err := s.chain.Search(c, docQuery.ToQuery(), docQuery.GetAttrQueries())
if err != nil {
c.String(500, fmt.Sprintf("search document error: %s", err))
return
}

var docWithAttrs []DocumentWithAttr
ids := []string{}
for _, doc := range docs {
ids = append(ids, doc.EntryId)
}

allAttrs, err := s.chain.ListDocumentAttrs(c, namespace, ids)
allAttrs, err := s.chain.ListDocumentAttrs(c, docQuery.Namespace, ids)
if err != nil {
c.String(500, fmt.Sprintf("list document attrs error: %s", err))
return
Expand All @@ -214,7 +155,6 @@ func (s *HttpServer) filter() gin.HandlerFunc {
attrsMap[attr.EntryId] = append(attrsMap[attr.EntryId], attr)
}

var docWithAttrs []DocumentWithAttr
for _, document := range docs {
docWithAttr := DocumentWithAttr{Document: document}
attrs := attrsMap[document.EntryId]
Expand All @@ -233,11 +173,84 @@ func (s *HttpServer) filter() gin.HandlerFunc {
}
docWithAttrs = append(docWithAttrs, docWithAttr)
}

c.JSON(200, docWithAttrs)
}
}

func getFilterQuery(c *gin.Context) *DocQuery {
namespace := c.Param("namespace")
page, err := strconv.Atoi(c.DefaultQuery("page", "0"))
if err != nil {
c.String(400, fmt.Sprintf("invalid page number: %s", c.Query("page")))
return nil
}
pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
if err != nil {
c.String(400, fmt.Sprintf("invalid pagesize: %s", c.Query("page")))
return nil
}

docQuery := DocQuery{
Namespace: namespace,
Source: c.Query("source"),
WebUrl: c.Query("webUrl"),
ParentID: c.Query("parentID"),
Search: c.Query("search"),
HitsPerPage: int64(pageSize),
Page: int64(page),
Sort: c.DefaultQuery("sort", "createdAt"),
Desc: c.DefaultQuery("desc", "false") == "true",
}

createAtStart := c.Query("createAtStart")
if createAtStart != "" {
createAtStartTimestamp, err := strconv.Atoi(createAtStart)
if err != nil {
c.String(400, fmt.Sprintf("invalid createAtStart: %s", c.Query("page")))
return nil
}
docQuery.CreatedAtStart = utils.ToPtr(int64(createAtStartTimestamp))
}
createAtEnd := c.Query("createAtEnd")
if createAtEnd != "" {
createAtEndTimestamp, err := strconv.Atoi(createAtEnd)
if err != nil {
c.String(400, fmt.Sprintf("invalid createAtEnd: %s", c.Query("page")))
return nil
}
docQuery.ChangedAtEnd = utils.ToPtr(int64(createAtEndTimestamp))
}
updatedAtStart := c.Query("updatedAtStart")
if updatedAtStart != "" {
updatedAtStartTimestamp, err := strconv.Atoi(updatedAtStart)
if err != nil {
c.String(400, fmt.Sprintf("invalid updatedAtStart: %s", c.Query("page")))
return nil
}
docQuery.ChangedAtStart = utils.ToPtr(int64(updatedAtStartTimestamp))
}
updatedAtEnd := c.Query("updatedAtEnd")
if updatedAtEnd != "" {
updatedAtEndTimestamp, err := strconv.Atoi(updatedAtEnd)
if err != nil {
c.String(400, fmt.Sprintf("invalid updatedAtEnd: %s", c.Query("page")))
return nil
}
docQuery.ChangedAtEnd = utils.ToPtr(int64(updatedAtEndTimestamp))
}
fuzzyName := c.Query("fuzzyName")
if fuzzyName != "" {
docQuery.FuzzyName = &fuzzyName
}
if c.Query("unRead") != "" {
docQuery.UnRead = utils.ToPtr(c.Query("unRead") == "true")
}
if c.Query("mark") != "" {
docQuery.Mark = utils.ToPtr(c.Query("mark") == "true")
}
return &docQuery
}

func (s *HttpServer) delete() gin.HandlerFunc {
return func(c *gin.Context) {
namespace := c.Param("namespace")
Expand Down
4 changes: 4 additions & 0 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@
package main

import (
"path"

"github.com/spf13/cobra"

"github.com/basenana/friday/cmd/apps"
"github.com/basenana/friday/config"
"github.com/basenana/friday/pkg/utils/logger"
)

Expand All @@ -36,6 +39,7 @@ func init() {

RootCmd.AddCommand(apps.ServeCmd)
RootCmd.AddCommand(apps.AgentCmd)
RootCmd.PersistentFlags().StringVar(&config.FilePath, "config", path.Join(config.LocalUserPath(), config.DefaultConfigBase), "friday config file")
}

func main() {
Expand Down
3 changes: 2 additions & 1 deletion pkg/service/chain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package service

import (
"context"
"fmt"

"go.uber.org/zap"

Expand Down Expand Up @@ -64,7 +65,7 @@ func (c *Chain) Store(ctx context.Context, document *doc.Document) error {
return err
} else if d != nil {
c.Log.Debugf("document already exists: %+v", d.String())
return nil
return fmt.Errorf("document already exists: %+v", d.String())
}
for _, plugin := range c.Plugins {
err := plugin.Run(ctx, document)
Expand Down
68 changes: 41 additions & 27 deletions pkg/store/docstore/meili.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"github.com/basenana/friday/config"
"github.com/basenana/friday/pkg/models/doc"
"github.com/basenana/friday/pkg/utils"
"github.com/basenana/friday/pkg/utils/logger"
)

Expand Down Expand Up @@ -56,20 +57,41 @@ func NewMeiliClient(conf config.Config) (DocStoreInterface, error) {
index: index,
client: client,
}
return meiliClient, meiliClient.init()
}

func (c *MeiliClient) init() error {
filterableAttrs := append(doc.DocFilterableAttrs, doc.DocAttrFilterableAttrs...)
t, err := client.Index(conf.MeiliConfig.Index).UpdateFilterableAttributes(&filterableAttrs)

attrs, err := c.index.GetFilterableAttributes()
if err != nil {
return nil, err
return err
}
if err = meiliClient.wait(context.TODO(), t.TaskUID); err != nil {
return nil, err
if !utils.Equal(filterableAttrs, attrs) {
t, err := c.index.UpdateFilterableAttributes(&filterableAttrs)
if err != nil {
return err
}
if err = c.wait(context.TODO(), t.TaskUID); err != nil {
return err
}
}

sortAttrs := doc.DocSortAttrs
t, err = client.Index(conf.MeiliConfig.Index).UpdateSortableAttributes(&sortAttrs)
crtSortAttrs, err := c.index.GetSortableAttributes()
if err != nil {
return nil, err
return err
}
return meiliClient, meiliClient.wait(context.TODO(), t.TaskUID)
if !utils.Equal(sortAttrs, crtSortAttrs) {
t, err := c.index.UpdateSortableAttributes(&sortAttrs)
if err != nil {
return err
}
if err = c.wait(context.TODO(), t.TaskUID); err != nil {
return err
}
}
return nil
}

func (c *MeiliClient) Store(ctx context.Context, docPtr doc.DocPtrInterface) error {
Expand All @@ -79,11 +101,9 @@ func (c *MeiliClient) Store(ctx context.Context, docPtr doc.DocPtrInterface) err
c.log.Error(err)
return err
}
go func() {
if err := c.wait(ctx, task.TaskUID); err != nil {
c.log.Errorf("store document with entryId %s error: %s", docPtr.EntryID(), err)
}
}()
if err := c.wait(ctx, task.TaskUID); err != nil {
c.log.Errorf("store document with entryId %s error: %s", docPtr.EntryID(), err)
}
return nil
}

Expand Down Expand Up @@ -134,11 +154,9 @@ func (c *MeiliClient) Update(ctx context.Context, document *doc.Document) error
c.log.Error(err)
return err
}
go func() {
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("update document %s error: %s", document.ID, err)
}
}()
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("update document %s error: %s", document.ID, err)
}
return nil
}

Expand All @@ -149,11 +167,9 @@ func (c *MeiliClient) Delete(ctx context.Context, docId string) error {
c.log.Error(err)
return err
}
go func() {
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("delete document %s error: %s", docId, err)
}
}()
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("delete document %s error: %s", docId, err)
}
return nil
}

Expand All @@ -169,11 +185,9 @@ func (c *MeiliClient) DeleteByFilter(ctx context.Context, aqs doc.DocumentAttrQu
c.log.Error(err)
return err
}
go func() {
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("delete document by filter error: %s", err)
}
}()
if err := c.wait(ctx, t.TaskUID); err != nil {
c.log.Errorf("delete document by filter error: %s", err)
}
return nil
}

Expand Down
29 changes: 29 additions & 0 deletions pkg/utils/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,35 @@

package utils

import (
"reflect"
"sort"
)

func ToPtr[T any](t T) *T {
return &t
}

func Equal(a []string, b *[]string) bool {
if b == nil {
return false
}
aa := deDup(a)
bb := deDup(*b)
sort.Sort(sort.StringSlice(aa))
sort.Sort(sort.StringSlice(bb))
return reflect.DeepEqual(aa, bb)
}

func deDup(res []string) []string {
uniqueMap := make(map[string]bool)
result := []string{}

for _, str := range res {
if _, exists := uniqueMap[str]; !exists {
uniqueMap[str] = true
result = append(result, str)
}
}
return result
}

0 comments on commit c98e5eb

Please sign in to comment.