From 3a9be4bed19ccecfa88dc88e5890dfe0b6440d62 Mon Sep 17 00:00:00 2001 From: Daniel Chalef <131175+danielchalef@users.noreply.github.com> Date: Sun, 15 Oct 2023 07:25:22 -0700 Subject: [PATCH 1/4] mmr memory search --- go.mod | 7 +- go.sum | 20 ++-- pkg/models/search.go | 24 +++-- pkg/search/mmr.go | 121 ++++++++--------------- pkg/search/mmr_test.go | 50 ++++++++++ pkg/store/postgres/search_memory.go | 68 ++++++++++--- pkg/store/postgres/search_memory_test.go | 12 ++- 7 files changed, 189 insertions(+), 113 deletions(-) create mode 100644 pkg/search/mmr_test.go diff --git a/go.mod b/go.mod index 9e0266c0..59d99a49 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/uptrace/bun v1.1.16 github.com/uptrace/bun/dialect/pgdialect v1.1.16 github.com/uptrace/bun/driver/pgdriver v1.1.16 - gonum.org/v1/gonum v0.14.0 ) require ( @@ -40,12 +39,14 @@ require ( github.com/tmc/langchaingo v0.0.0-20230929160525-e16b77704b8d github.com/uptrace/bun/dbfixture v1.1.16 github.com/uptrace/bun/extra/bundebug v1.1.16 + github.com/viterin/vek v0.4.2 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect + github.com/chewxy/math32 v1.10.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect @@ -92,13 +93,15 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/sv-tools/openapi v0.2.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/viterin/partial v1.1.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect mellium.im/sasl v0.3.1 // indirect diff --git a/go.sum b/go.sum index 89a65b87..8ea0b329 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5A github.com/brianvoe/gofakeit/v6 v6.23.2 h1:lVde18uhad5wII/f5RMVFLtdQNE0HaGFuBUXmYKk8i8= github.com/brianvoe/gofakeit/v6 v6.23.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU= +github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= github.com/chi-middleware/logrus-logger v0.2.0 h1:Do3vcVSRsLh7zSRKxsVg5Kr5//rTqytwprCR1HzVqT8= github.com/chi-middleware/logrus-logger v0.2.0/go.mod h1:ie/rvKsXrtqqsnJd3qtSEnLxgCs1I758WYmHdv6CRt0= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -347,6 +349,10 @@ github.com/uptrace/bun/driver/pgdriver v1.1.16 h1:b/NiSXk6Ldw7KLfMLbOqIkm4odHd7Q github.com/uptrace/bun/driver/pgdriver v1.1.16/go.mod h1:Rmfbc+7lx1z/umjMyAxkOHK81LgnGj71XC5YpA6k1vU= github.com/uptrace/bun/extra/bundebug v1.1.16 h1:SgicRQGtnjhrIhlYOxdkOm1Em4s6HykmT3JblHnoTBM= github.com/uptrace/bun/extra/bundebug v1.1.16/go.mod h1:SkiOkfUirBiO1Htc4s5bQKEq+JSeU1TkBVpMsPz2ePM= +github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E= +github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= +github.com/viterin/vek v0.4.2 h1:Vyv04UjQT6gcjEFX82AS9ocgNbAJqsHviheIBdPlv5U= +github.com/viterin/vek v0.4.2/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= @@ -389,8 +395,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -416,8 +422,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -599,14 +605,12 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0= -gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/pkg/models/search.go b/pkg/models/search.go index 0ad2adf9..37805a4c 100644 --- a/pkg/models/search.go +++ b/pkg/models/search.go @@ -1,15 +1,25 @@ package models +type SearchType string + +const ( + SearchTypeSimilarity SearchType = "similarity" + SearchTypeMMR SearchType = "mmr" +) + type MemorySearchResult struct { - Message *Message `json:"message"` - Summary *Summary `json:"summary"` // reserved for future use - Metadata map[string]interface{} `json:"metadata,omitempty"` - Dist float64 `json:"dist"` + Message *Message `json:"message"` + Summary *Summary `json:"summary"` // reserved for future use + Metadata map[string]interface{} `json:"metadata,omitempty"` + Dist float64 `json:"dist"` + Embedding []float32 `json:"embedding"` } type MemorySearchPayload struct { - Text string `json:"text"` - Metadata map[string]interface{} `json:"metadata,omitempty"` + Text string `json:"text"` + Metadata map[string]interface{} `json:"metadata,omitempty"` + Type SearchType `json:"type"` + MMRLambda float32 `json:"mmr_lambda,omitempty"` } type DocumentSearchPayload struct { @@ -17,6 +27,8 @@ type DocumentSearchPayload struct { Text string `json:"text,omitempty"` Embedding []float32 `json:"embedding,omitempty"` Metadata map[string]interface{} `json:"metadata,omitempty"` + Type SearchType `json:"type"` + MMRLambda float32 `json:"mmr_lambda,omitempty"` } type DocumentSearchResult struct { diff --git a/pkg/search/mmr.go b/pkg/search/mmr.go index 430dd6e0..dc1bf9f6 100644 --- a/pkg/search/mmr.go +++ b/pkg/search/mmr.go @@ -1,108 +1,79 @@ package search import ( + "errors" "fmt" "math" - "gonum.org/v1/gonum/floats" - - "gonum.org/v1/gonum/mat" + "github.com/getzep/zep/internal" + "github.com/viterin/vek" + "github.com/viterin/vek/vek32" ) -// CosineSimilarity calculates the cosine similarity between two vectors. -// The vectors must be of the same length. -func CosineSimilarity(X, Y *mat.Dense) (*mat.Dense, error) { // nolint: gocritic - rX, cX := X.Dims() - rY, cY := Y.Dims() - - if rX == 0 || rY == 0 { - return mat.NewDense(0, 0, nil), nil - } - - if cX != cY { - return nil, fmt.Errorf( - "number of columns in X and Y must be the same. X has shape [%d, %d] and Y has shape [%d, %d]", - rX, - cX, - rY, - cY, - ) - } - - Xnorm := mat.NewVecDense(rX, nil) - Ynorm := mat.NewVecDense(rY, nil) - - for i := 0; i < rX; i++ { - Xnorm.SetVec(i, mat.Norm(X.RowView(i), 2)) - } - - for i := 0; i < rY; i++ { - Ynorm.SetVec(i, mat.Norm(Y.RowView(i), 2)) - } +var log = internal.GetLogger() - var XT mat.Dense - XT.CloneFrom(X.T()) - - similarity := mat.NewDense(rX, rY, nil) - similarity.Product(X, &XT) +func init() { + log.Infof("MMR acceleration status: %v", vek.Info()) +} - for i := 0; i < rX; i++ { - for j := 0; j < rY; j++ { - val := similarity.At(i, j) / (Xnorm.AtVec(i) * Ynorm.AtVec(j)) - if math.IsNaN(val) || math.IsInf(val, 0) { - val = 0.0 +// pairwiseCosineSimilarity takes two matrices of vectors and returns a matrix, where +// the value at [i][j] is the cosine similarity between the ith vector in matrix1 and +// the jth vector in matrix2. +func pairwiseCosineSimilarity(matrix1 [][]float32, matrix2 [][]float32) ([][]float32, error) { + result := make([][]float32, len(matrix1)) + for i, vec1 := range matrix1 { + result[i] = make([]float32, len(matrix2)) + for j, vec2 := range matrix2 { + if len(vec1) != len(vec2) { + return nil, fmt.Errorf("vector lengths do not match: %d != %d", len(vec1), len(vec2)) } - similarity.Set(i, j, val) + result[i][j] = vek32.CosineSimilarity(vec1, vec2) } } - - return similarity, nil + return result, nil } // MaximalMarginalRelevance implements the Maximal Marginal Relevance algorithm. // It takes a query embedding, a list of embeddings, a lambda multiplier, and a // number of results to return. It returns a list of indices of the embeddings // that are most relevant to the query. -// This is a relatively naive and unoptimized implementation of MMR. :-/ // See https://www.cs.cmu.edu/~jgc/publication/The_Use_MMR_Diversity_Based_LTMIR_1998.pdf -func MaximalMarginalRelevance( - queryEmbedding *mat.Dense, - embeddingList *mat.Dense, - lambdaMult float64, - k int, -) ([]int, error) { - rEmbed, _ := embeddingList.Dims() - if k <= 0 || rEmbed == 0 { +// Implementation borrowed from LangChain +// https://github.com/langchain-ai/langchain/blob/4a2f0c51a116cc3141142ea55254e270afb6acde/libs/langchain/langchain/vectorstores/utils.py#L23 +func MaximalMarginalRelevance(queryEmbedding []float32, embeddingList [][]float32, lambdaMult float32, k int) ([]int, error) { + // if either k or the length of the embedding list is 0, return an empty list + if min(k, len(embeddingList)) <= 0 { return []int{}, nil } - var mostSimilar int - var bestScore float64 - var idxToAdd int + // We expect the query embedding and the embeddings in the list to have the same width + if len(queryEmbedding) != len(embeddingList[0]) { + return []int{}, errors.New("query embedding width does not match embedding vector width") + } - similarityToQuery, err := CosineSimilarity(queryEmbedding, embeddingList) + similarityToQueryMatrix, err := pairwiseCosineSimilarity([][]float32{queryEmbedding}, embeddingList) if err != nil { return nil, err } - mostSimilar = floats.MaxIdx(similarityToQuery.RawMatrix().Data) + similarityToQuery := similarityToQueryMatrix[0] + + mostSimilar := vek32.ArgMax(similarityToQuery) idxs := []int{mostSimilar} - selected := mat.DenseCopyOf(embeddingList.RowView(mostSimilar)) + selected := [][]float32{embeddingList[mostSimilar]} - for len(idxs) < min(k, rEmbed) { - bestScore = math.Inf(-1) - idxToAdd = -1 - r, c := selected.Dims() - selectedTransposed := mat.NewDense(c, r, nil) - selectedTransposed.CloneFrom(selected.T()) - similarityToSelected, err := CosineSimilarity(embeddingList, selectedTransposed) + for len(idxs) < min(k, len(embeddingList)) { + var bestScore float32 = -math.MaxFloat32 + idxToAdd := -1 + similarityToSelected, err := pairwiseCosineSimilarity(embeddingList, selected) if err != nil { return nil, err } - for i, queryScore := range similarityToQuery.RawMatrix().Data { + + for i, queryScore := range similarityToQuery { if contains(idxs, i) { continue } - redundantScore := floats.Max(similarityToSelected.RawMatrix().Data) + redundantScore := vek32.Max(similarityToSelected[i]) equationScore := lambdaMult*queryScore - (1-lambdaMult)*redundantScore if equationScore > bestScore { bestScore = equationScore @@ -110,11 +81,12 @@ func MaximalMarginalRelevance( } } idxs = append(idxs, idxToAdd) - selected.Stack(selected, embeddingList.RowView(idxToAdd)) + selected = append(selected, embeddingList[idxToAdd]) } return idxs, nil } +// contains returns true if the slice contains the value func contains(slice []int, val int) bool { for _, item := range slice { if item == val { @@ -123,10 +95,3 @@ func contains(slice []int, val int) bool { } return false } - -func min(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/pkg/search/mmr_test.go b/pkg/search/mmr_test.go new file mode 100644 index 00000000..918b7d7f --- /dev/null +++ b/pkg/search/mmr_test.go @@ -0,0 +1,50 @@ +package search + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMaximalMarginalRelevance(t *testing.T) { + // Test case for mismatched vector widths + t.Run("MismatchedVectorWidths", func(t *testing.T) { + queryEmbedding := []float32{0.1, 0.2, 0.3, 0.4, 0.5} + embeddingList := [][]float32{ + {0.1, 0.2, 0.3}, + {0.2, 0.3, 0.4, 0.5, 0.5}, + } + _, err := MaximalMarginalRelevance(queryEmbedding, embeddingList, 0.5, 2) + assert.Error(t, err) + }) + + // Test case for checking ranking + t.Run("Ranking", func(t *testing.T) { + queryEmbedding := []float32{0.1, 0.2, 0.3, 0.4, 0.5} + embeddingList := [][]float32{ + {0.1, 0.2, 0.3, 0.4, 0.4}, + {0.2, 0.3, 0.4, 0.5, 0.5}, + {0.1, 0.2, 0.3, 0.4, 0.6}, + {0.1, 0.0, 0.0, 0.0, 0.0}, + {0.2, 0.0, 0.0, 0.0, 0.0}, + } + expected := []int{2, 1} + result, err := MaximalMarginalRelevance(queryEmbedding, embeddingList, 0.5, 2) + assert.NoError(t, err) + assert.Equal(t, expected, result) + }) + + // Test case for modifying lambda + t.Run("LambdaModification", func(t *testing.T) { + queryEmbedding := []float32{0.1, 0.2, 0.3, 0.4, 0.5} + embeddingList := [][]float32{ + {0.1, 0.2, 0.3, 0.4, 0.4}, + {0.2, 0.3, 0.4, 0.5, 0.5}, + {0.1, 0.2, 0.3, 0.4, 0.6}, + } + expected := []int{2, 0} + result, err := MaximalMarginalRelevance(queryEmbedding, embeddingList, 1.0, 2) + assert.NoError(t, err) + assert.Equal(t, expected, result) + }) +} diff --git a/pkg/store/postgres/search_memory.go b/pkg/store/postgres/search_memory.go index 213189f1..01710ad1 100644 --- a/pkg/store/postgres/search_memory.go +++ b/pkg/store/postgres/search_memory.go @@ -6,16 +6,17 @@ import ( "errors" "math" - "github.com/sirupsen/logrus" - "github.com/getzep/zep/pkg/llms" "github.com/getzep/zep/pkg/models" + "github.com/getzep/zep/pkg/search" "github.com/getzep/zep/pkg/store" "github.com/pgvector/pgvector-go" "github.com/uptrace/bun" ) const DefaultMemorySearchLimit = 10 +const DefaultMMRMultiplier = 2 +const DefaultMMRLambda = 0.5 type JSONQuery struct { JSONPath string `json:"jsonpath"` @@ -31,8 +32,6 @@ func searchMessages( query *models.MemorySearchPayload, limit int, ) ([]models.MemorySearchResult, error) { - logrus.Debugf("searchMessages called for session %s", sessionID) - if query == nil || appState == nil { return nil, store.NewStorageError("nil query or appState received", nil) } @@ -41,7 +40,15 @@ func searchMessages( return nil, store.NewStorageError("empty query", errors.New("empty query")) } - dbQuery := buildMessagesSelectQuery(ctx, appState, db, query) + dbQuery := buildMessagesSelectQuery(ctx, db, query) + var err error + var queryEmbedding []float32 + if query.Text != "" { + dbQuery, queryEmbedding, err = addMessagesVectorColumn(ctx, appState, dbQuery, query.Text) + if err != nil { + return nil, store.NewStorageError("error adding vector column", err) + } + } if len(query.Metadata) > 0 { var err error dbQuery, err = applyMessagesMetadataFilter(dbQuery, query.Metadata) @@ -61,7 +68,17 @@ func searchMessages( if limit == 0 { limit = DefaultMemorySearchLimit } - dbQuery = dbQuery.Limit(limit) + + // If we're using MMR, we need to return more results than the limit so we can + // rerank them. + if query.Type == models.SearchTypeMMR { + if query.MMRLambda == 0 { + query.MMRLambda = DefaultMMRLambda + } + dbQuery = dbQuery.Limit(limit * DefaultMMRMultiplier) + } else { + dbQuery = dbQuery.Limit(limit) + } results, err := executeMessagesSearchScan(ctx, dbQuery) if err != nil { @@ -69,14 +86,37 @@ func searchMessages( } filteredResults := filterValidMessageSearchResults(results, query.Metadata) - logrus.Debugf("searchMessages completed for session %s", sessionID) + + // If we're using MMR, rerank the results. + if query.Type == models.SearchTypeMMR { + filteredResults, err = rerankMMR(filteredResults, queryEmbedding, query.MMRLambda, limit) + if err != nil { + return nil, store.NewStorageError("error applying mmr", err) + } + } return filteredResults, nil } +// rerankMMR reranks the results using the Maximal Marginal Relevance algorithm +func rerankMMR(results []models.MemorySearchResult, queryEmbedding []float32, lambda float32, limit int) ([]models.MemorySearchResult, error) { + embeddingList := make([][]float32, len(results)) + for i, result := range results { + embeddingList[i] = result.Embedding + } + rerankedIdxs, err := search.MaximalMarginalRelevance(queryEmbedding, embeddingList, lambda, limit) + if err != nil { + return nil, store.NewStorageError("error applying mmr", err) + } + rerankedResults := make([]models.MemorySearchResult, len(rerankedIdxs)) + for i, idx := range rerankedIdxs { + rerankedResults[i] = results[idx] + } + return rerankedResults, nil +} + func buildMessagesSelectQuery( ctx context.Context, - appState *models.AppState, db *bun.DB, query *models.MemorySearchPayload, ) *bun.SelectQuery { @@ -90,8 +130,8 @@ func buildMessagesSelectQuery( ColumnExpr("m.metadata AS message__metadata"). ColumnExpr("m.token_count AS message__token_count") - if query.Text != "" { - dbQuery, _ = addMessagesVectorColumn(ctx, appState, dbQuery, query.Text) + if query.Type == models.SearchTypeMMR { + dbQuery = dbQuery.ColumnExpr("me.embedding AS embedding") } return dbQuery @@ -170,18 +210,18 @@ func addMessagesVectorColumn( appState *models.AppState, q *bun.SelectQuery, queryText string, -) (*bun.SelectQuery, error) { +) (*bun.SelectQuery, []float32, error) { documentType := "message" model, err := llms.GetEmbeddingModel(appState, documentType) if err != nil { - return nil, store.NewStorageError("failed to get message embedding model", err) + return nil, nil, store.NewStorageError("failed to get message embedding model", err) } e, err := llms.EmbedTexts(ctx, appState, model, documentType, []string{queryText}) if err != nil { - return nil, store.NewStorageError("failed to embed query", err) + return nil, nil, store.NewStorageError("failed to embed query", err) } vector := pgvector.NewVector(e[0]) - return q.ColumnExpr("(embedding <#> ?) * -1 AS dist", vector), nil + return q.ColumnExpr("(embedding <#> ?) * -1 AS dist", vector), e[0], nil } diff --git a/pkg/store/postgres/search_memory_test.go b/pkg/store/postgres/search_memory_test.go index bf05144d..72ba8d20 100644 --- a/pkg/store/postgres/search_memory_test.go +++ b/pkg/store/postgres/search_memory_test.go @@ -39,16 +39,18 @@ func TestMemorySearch(t *testing.T) { query string limit int expectedErrorText string + searchType models.SearchType }{ - {"Empty Query", "", 0, "empty query"}, - {"Non-empty Query", "travel", 0, ""}, - {"Limit 0", "travel", 0, ""}, - {"Limit 5", "travel", 5, ""}, + {"Empty Query", "", 0, "empty query", models.SearchTypeSimilarity}, + {"Non-empty Query", "travel", 0, "", models.SearchTypeSimilarity}, + {"Limit 0", "travel", 0, "", models.SearchTypeSimilarity}, + {"Limit 5", "travel", 5, "", models.SearchTypeSimilarity}, + {"MMR Query", "travel", 5, "", models.SearchTypeMMR}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - q := models.MemorySearchPayload{Text: tc.query} + q := models.MemorySearchPayload{Text: tc.query, Type: tc.searchType} expectedLastN := tc.limit if expectedLastN == 0 { expectedLastN = 10 // Default value From cdf1f276e52aa065bfe82f55391fa88b872000ee Mon Sep 17 00:00:00 2001 From: Daniel Chalef <131175+danielchalef@users.noreply.github.com> Date: Sun, 15 Oct 2023 08:03:20 -0700 Subject: [PATCH 2/4] remove withMMR from documents --- pkg/models/documentstore.go | 1 - pkg/search/mmr.go | 2 +- pkg/server/apihandlers/document_handlers.go | 20 +------------------- pkg/store/postgres/document_search.go | 2 -- pkg/store/postgres/document_search_test.go | 1 - pkg/store/postgres/documents.go | 2 -- pkg/store/postgres/documentstore.go | 3 +-- 7 files changed, 3 insertions(+), 28 deletions(-) diff --git a/pkg/models/documentstore.go b/pkg/models/documentstore.go index b22518f5..6c013d67 100644 --- a/pkg/models/documentstore.go +++ b/pkg/models/documentstore.go @@ -70,7 +70,6 @@ type DocumentStore[T any] interface { ctx context.Context, query *DocumentSearchPayload, limit int, - withMMR bool, // withMMR is used to enable/disable the Maximal Marginal Relevance algorithm for search results. pageNumber int, pageSize int, ) (*DocumentSearchResultPage, error) diff --git a/pkg/search/mmr.go b/pkg/search/mmr.go index dc1bf9f6..db60a775 100644 --- a/pkg/search/mmr.go +++ b/pkg/search/mmr.go @@ -39,7 +39,7 @@ func pairwiseCosineSimilarity(matrix1 [][]float32, matrix2 [][]float32) ([][]flo // that are most relevant to the query. // See https://www.cs.cmu.edu/~jgc/publication/The_Use_MMR_Diversity_Based_LTMIR_1998.pdf // Implementation borrowed from LangChain -// https://github.com/langchain-ai/langchain/blob/4a2f0c51a116cc3141142ea55254e270afb6acde/libs/langchain/langchain/vectorstores/utils.py#L23 +// https://github.com/langchain-ai/langchain/blob/4a2f0c51a116cc3141142ea55254e270afb6acde/libs/langchain/langchain/vectorstores/utils.py func MaximalMarginalRelevance(queryEmbedding []float32, embeddingList [][]float32, lambdaMult float32, k int) ([]int, error) { // if either k or the length of the embedding list is 0, return an empty list if min(k, len(embeddingList)) <= 0 { diff --git a/pkg/server/apihandlers/document_handlers.go b/pkg/server/apihandlers/document_handlers.go index d6a03491..daaa234a 100644 --- a/pkg/server/apihandlers/document_handlers.go +++ b/pkg/server/apihandlers/document_handlers.go @@ -783,7 +783,6 @@ func CreateCollectionIndexHandler(appState *models.AppState) http.HandlerFunc { // @Produce json // @Param collectionName path string true "Name of the Document Collection" // @Param limit query int false "Limit the number of returned documents" -// @Param mmr query bool false "Use MMR to rerank the search results. Not Implemented" // @Param searchPayload body models.DocumentSearchPayload true "Search criteria" // @Success 200 {object} []models.Document "OK" // @Failure 400 {object} APIError "Bad Request" @@ -812,23 +811,6 @@ func SearchDocumentsHandler(appState *models.AppState) http.HandlerFunc { return } - // disabled for now - withMMRStr := r.URL.Query().Get("mmr") - withMMR := false - if withMMRStr != "" { - _, err = strconv.ParseBool(withMMRStr) - if err != nil { - handlertools.RenderError(w, err, http.StatusBadRequest) - return - } - handlertools.RenderError( - w, - errors.New("MMR not yet implemented"), - http.StatusBadRequest, - ) - return - } - var searchPayload models.DocumentSearchPayload if err := json.NewDecoder(r.Body).Decode(&searchPayload); err != nil { handlertools.RenderError(w, err, http.StatusBadRequest) @@ -837,7 +819,7 @@ func SearchDocumentsHandler(appState *models.AppState) http.HandlerFunc { searchPayload.CollectionName = collectionName - results, err := store.SearchCollection(r.Context(), &searchPayload, limit, withMMR, 0, 0) + results, err := store.SearchCollection(r.Context(), &searchPayload, limit, 0, 0) if err != nil { if errors.Is(err, models.ErrNotFound) { handlertools.RenderError(w, err, http.StatusNotFound) diff --git a/pkg/store/postgres/document_search.go b/pkg/store/postgres/document_search.go index 9e2972b4..07efba4a 100644 --- a/pkg/store/postgres/document_search.go +++ b/pkg/store/postgres/document_search.go @@ -26,7 +26,6 @@ func newDocumentSearchOperation( searchPayload *models.DocumentSearchPayload, collection *models.DocumentCollection, limit int, - withMMR bool, ) *documentSearchOperation { if limit <= 0 { limit = DefaultDocumentSearchLimit @@ -39,7 +38,6 @@ func newDocumentSearchOperation( searchPayload: searchPayload, collection: collection, limit: limit, - withMMR: withMMR, } } diff --git a/pkg/store/postgres/document_search_test.go b/pkg/store/postgres/document_search_test.go index 18245e74..5777e9e6 100644 --- a/pkg/store/postgres/document_search_test.go +++ b/pkg/store/postgres/document_search_test.go @@ -75,7 +75,6 @@ func TestDocumentSearchWithIndexEndToEnd(t *testing.T) { ctx, searchPayload, limit, - false, 0, 0, ) diff --git a/pkg/store/postgres/documents.go b/pkg/store/postgres/documents.go index c7a331d3..c2583491 100644 --- a/pkg/store/postgres/documents.go +++ b/pkg/store/postgres/documents.go @@ -463,7 +463,6 @@ func (dc *DocumentCollectionDAO) DeleteDocumentsByUUID( func (dc *DocumentCollectionDAO) SearchDocuments(ctx context.Context, query *models.DocumentSearchPayload, limit int, - withMMR bool, pageNumber int, pageSize int) (*models.DocumentSearchResultPage, error) { // TODO: implement pagination @@ -493,7 +492,6 @@ func (dc *DocumentCollectionDAO) SearchDocuments(ctx context.Context, query, &dc.DocumentCollection, limit, - withMMR, ) results, err := search.Execute() diff --git a/pkg/store/postgres/documentstore.go b/pkg/store/postgres/documentstore.go index 0b0a447e..d277e8c0 100644 --- a/pkg/store/postgres/documentstore.go +++ b/pkg/store/postgres/documentstore.go @@ -295,7 +295,6 @@ func (ds *DocumentStore) SearchCollection( ctx context.Context, query *models.DocumentSearchPayload, limit int, - withMMR bool, pageNumber int, pageSize int, ) (*models.DocumentSearchResultPage, error) { @@ -305,7 +304,7 @@ func (ds *DocumentStore) SearchCollection( models.DocumentCollection{Name: query.CollectionName}, ) - results, err := collectionDAO.SearchDocuments(ctx, query, limit, withMMR, pageNumber, pageSize) + results, err := collectionDAO.SearchDocuments(ctx, query, limit, pageNumber, pageSize) if err != nil { return nil, fmt.Errorf("failed to search collection: %w", err) } From e4b4a9ce9621419eb029a309fdcd3d8a0fdde7d9 Mon Sep 17 00:00:00 2001 From: Daniel Chalef <131175+danielchalef@users.noreply.github.com> Date: Sun, 15 Oct 2023 08:04:48 -0700 Subject: [PATCH 3/4] update swagger --- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 24 ++++++++++++++++++++---- pkg/server/routes.go | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index fc492a05..69c943d9 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -5,7 +5,7 @@ package docs import "github.com/swaggo/swag/v2" const docTemplate = `{ - "schemes": {{ marshal .Schemes }},"swagger":"2.0","info":{"description":"{{escape .Description}}","title":"{{.Title}}","contact":{},"license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0.html"},"version":"{{.Version}}"},"host":"{{.Host}}","basePath":"{{.BasePath}}","paths":{"/api/v1/collection":{"get":{"security":[{"Bearer":[]}],"description":"Returns a list of all DocumentCollections.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a list of DocumentCollections","responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentCollectionResponse"}}}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}":{"get":{"security":[{"Bearer":[]}],"description":"Returns a DocumentCollection if it exists.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentCollectionResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, an error will be returned.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates a new DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, it will be overwritten.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Deletes a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Updates a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document":{"post":{"security":[{"Bearer":[]}],"description":"Creates Documents in a specified DocumentCollection and returns their UUIDs.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Creates Multiple Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be created","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.CreateDocumentRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"string"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchDelete":{"post":{"security":[{"Bearer":[]}],"description":"Deletes specified Documents from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Deletes Documents from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs of the Documents to be deleted","name":"documentUUIDs","in":"body","required":true,"schema":{"type":"array","items":{"type":"string"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchGet":{"post":{"security":[{"Bearer":[]}],"description":"Returns Documents from a DocumentCollection specified by UUID or ID.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Gets Documents from a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs and IDs of the Documents to be fetched","name":"documentRequest","in":"body","required":true,"schema":{"$ref":"#/definitions/models.GetDocumentListRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentResponse"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchUpdate":{"patch":{"security":[{"Bearer":[]}],"description":"Updates Documents in a specified DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Updates Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be updated","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.UpdateDocumentListRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/uuid/{documentUUID}":{"get":{"security":[{"Bearer":[]}],"description":"Returns specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Gets a Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"Delete specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Delete Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be deleted","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Document Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Updates a Document in a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true},{"description":"Document to be updated","name":"document","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/index/create":{"post":{"security":[{"Bearer":[]}],"description":"Creates an index for the specified DocumentCollection to improve query performance.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates an index for a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"boolean","description":"Force index creation, even if there are too few documents to index","name":"force","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/search":{"post":{"security":[{"Bearer":[]}],"description":"Searches Documents in a DocumentCollection based on provided search criteria.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Searches Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"integer","description":"Limit the number of returned documents","name":"limit","in":"query"},{"type":"boolean","description":"Use MMR to rerank the search results. Not Implemented","name":"mmr","in":"query"},{"description":"Search criteria","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.DocumentSearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Document"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions":{"get":{"security":[{"Bearer":[]}],"description":"get all sessions with optional limit and cursor for pagination","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns all sessions","parameters":[{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"type":"integer","description":"Cursor for pagination)","name":"cursor","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateSessionRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}":{"get":{"security":[{"Bearer":[]}],"description":"get session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns a session by ID","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateSessionRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/memory":{"get":{"security":[{"Bearer":[]}],"description":"get memory by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Returns a memory (latest summary and list of messages) for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Last N messages. Overrides memory_window configuration","name":"lastn","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Memory"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Add memory messages to a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Memory messages","name":"memoryMessages","in":"body","required":true,"schema":{"$ref":"#/definitions/models.Memory"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Delete memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/search":{"post":{"security":[{"Bearer":[]}],"description":"search memory messages by session id and query","consumes":["application/json"],"produces":["application/json"],"tags":["search"],"summary":"Search memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"description":"Search query","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.MemorySearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.MemorySearchResult"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user":{"get":{"security":[{"Bearer":[]}],"description":"list all users with pagination","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all users","parameters":[{"type":"integer","description":"Limit","name":"limit","in":"query"},{"type":"integer","description":"Cursor","name":"cursor","in":"query"}],"responses":{"200":{"description":"Successfully retrieved list of users","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.User"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Add a user","parameters":[{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateUserRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}":{"get":{"security":[{"Bearer":[]}],"description":"get user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Returns a user by ID","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Delete a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"update user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Update a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true},{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateUserRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}/sessions":{"get":{"security":[{"Bearer":[]}],"description":"list all sessions for a user by user id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all sessions for a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}}},"definitions":{"apihandlers.APIError":{"type":"object","properties":{"message":{"type":"string"}}},"models.CreateDocumentCollectionRequest":{"type":"object","required":["embedding_dimensions","is_auto_embedded","name"],"properties":{"description":{"type":"string","maxLength":1000},"embedding_dimensions":{"type":"integer","maximum":2000,"minimum":8},"is_auto_embedded":{"description":"these needs to be pointers so that we can distinguish between false and unset when validating","type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string","maxLength":40,"minLength":3}}},"models.CreateDocumentRequest":{"type":"object","properties":{"content":{"type":"string"},"document_id":{"type":"string","maxLength":100},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true}}},"models.CreateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"}}},"models.CreateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"}}},"models.Document":{"type":"object","properties":{"content":{"type":"string"},"createdAt":{"type":"string"},"deletedAt":{"type":"string"},"documentID":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"isEmbedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updatedAt":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentCollectionResponse":{"type":"object","properties":{"created_at":{"type":"string"},"description":{"type":"string"},"document_count":{"description":"Number of documents in the collection","type":"integer"},"document_embedded_count":{"description":"Number of documents with embeddings","type":"integer"},"embedding_dimensions":{"type":"integer"},"embedding_model_name":{"type":"string"},"is_auto_embedded":{"type":"boolean"},"is_indexed":{"type":"boolean"},"is_normalized":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentResponse":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"document_id":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"is_embedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentSearchPayload":{"type":"object","properties":{"collection_name":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true},"text":{"type":"string"}}},"models.GetDocumentListRequest":{"type":"object","properties":{"document_ids":{"type":"array","items":{"type":"string"}},"uuids":{"type":"array","items":{"type":"string"}}}},"models.Memory":{"type":"object","properties":{"messages":{"type":"array","items":{"$ref":"#/definitions/models.Message"}},"metadata":{"type":"object","additionalProperties":true},"summary":{"$ref":"#/definitions/models.Summary"}}},"models.MemorySearchPayload":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"text":{"type":"string"}}},"models.MemorySearchResult":{"type":"object","properties":{"dist":{"type":"number"},"message":{"$ref":"#/definitions/models.Message"},"metadata":{"type":"object","additionalProperties":true},"summary":{"description":"reserved for future use","allOf":[{"$ref":"#/definitions/models.Summary"}]}}},"models.Message":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"role":{"type":"string"},"token_count":{"type":"integer"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.Session":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"id":{"type":"integer"},"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"updated_at":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"},"uuid":{"type":"string"}}},"models.Summary":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"recent_message_uuid":{"description":"The most recent message UUID that was used to generate this summary","type":"string"},"token_count":{"type":"integer"},"uuid":{"type":"string"}}},"models.UpdateDocumentCollectionRequest":{"type":"object","properties":{"description":{"type":"string","maxLength":1000},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateDocumentListRequest":{"type":"object","required":["uuid"],"properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true},"uuid":{"type":"string"}}},"models.UpdateDocumentRequest":{"type":"object","properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"}}},"models.UpdateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"},"uuid":{"type":"string"}}},"models.User":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"email":{"type":"string"},"first_name":{"type":"string"},"id":{"type":"integer"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"user_id":{"type":"string"},"uuid":{"type":"string"}}}},"securityDefinitions":{"Bearer":{"description":"Type \"Bearer\" followed by a space and JWT token.","type":"apiKey","name":"Authorization","in":"header"}}}` + "schemes": {{ marshal .Schemes }},"swagger":"2.0","info":{"description":"{{escape .Description}}","title":"{{.Title}}","contact":{},"license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0.html"},"version":"{{.Version}}"},"host":"{{.Host}}","basePath":"{{.BasePath}}","paths":{"/api/v1/collection":{"get":{"security":[{"Bearer":[]}],"description":"Returns a list of all DocumentCollections.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a list of DocumentCollections","responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentCollectionResponse"}}}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}":{"get":{"security":[{"Bearer":[]}],"description":"Returns a DocumentCollection if it exists.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentCollectionResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, an error will be returned.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates a new DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, it will be overwritten.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Deletes a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Updates a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document":{"post":{"security":[{"Bearer":[]}],"description":"Creates Documents in a specified DocumentCollection and returns their UUIDs.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Creates Multiple Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be created","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.CreateDocumentRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"string"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchDelete":{"post":{"security":[{"Bearer":[]}],"description":"Deletes specified Documents from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Deletes Documents from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs of the Documents to be deleted","name":"documentUUIDs","in":"body","required":true,"schema":{"type":"array","items":{"type":"string"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchGet":{"post":{"security":[{"Bearer":[]}],"description":"Returns Documents from a DocumentCollection specified by UUID or ID.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Gets Documents from a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs and IDs of the Documents to be fetched","name":"documentRequest","in":"body","required":true,"schema":{"$ref":"#/definitions/models.GetDocumentListRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentResponse"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchUpdate":{"patch":{"security":[{"Bearer":[]}],"description":"Updates Documents in a specified DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Updates Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be updated","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.UpdateDocumentListRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/uuid/{documentUUID}":{"get":{"security":[{"Bearer":[]}],"description":"Returns specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Gets a Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"Delete specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Delete Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be deleted","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Document Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Updates a Document in a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true},{"description":"Document to be updated","name":"document","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/index/create":{"post":{"security":[{"Bearer":[]}],"description":"Creates an index for the specified DocumentCollection to improve query performance.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates an index for a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"boolean","description":"Force index creation, even if there are too few documents to index","name":"force","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/search":{"post":{"security":[{"Bearer":[]}],"description":"Searches Documents in a DocumentCollection based on provided search criteria.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Searches Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"integer","description":"Limit the number of returned documents","name":"limit","in":"query"},{"description":"Search criteria","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.DocumentSearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Document"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions":{"get":{"security":[{"Bearer":[]}],"description":"get all sessions with optional limit and cursor for pagination","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns all sessions","parameters":[{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"type":"integer","description":"Cursor for pagination)","name":"cursor","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateSessionRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}":{"get":{"security":[{"Bearer":[]}],"description":"get session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns a session by ID","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateSessionRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/memory":{"get":{"security":[{"Bearer":[]}],"description":"get memory by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Returns a memory (latest summary and list of messages) for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Last N messages. Overrides memory_window configuration","name":"lastn","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Memory"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Add memory messages to a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Memory messages","name":"memoryMessages","in":"body","required":true,"schema":{"$ref":"#/definitions/models.Memory"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Delete memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/search":{"post":{"security":[{"Bearer":[]}],"description":"search memory messages by session id and query","consumes":["application/json"],"produces":["application/json"],"tags":["search"],"summary":"Search memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"description":"Search query","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.MemorySearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.MemorySearchResult"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user":{"get":{"security":[{"Bearer":[]}],"description":"list all users with pagination","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all users","parameters":[{"type":"integer","description":"Limit","name":"limit","in":"query"},{"type":"integer","description":"Cursor","name":"cursor","in":"query"}],"responses":{"200":{"description":"Successfully retrieved list of users","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.User"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Add a user","parameters":[{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateUserRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}":{"get":{"security":[{"Bearer":[]}],"description":"get user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Returns a user by ID","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Delete a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"update user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Update a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true},{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateUserRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}/sessions":{"get":{"security":[{"Bearer":[]}],"description":"list all sessions for a user by user id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all sessions for a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}}},"definitions":{"apihandlers.APIError":{"type":"object","properties":{"message":{"type":"string"}}},"models.CreateDocumentCollectionRequest":{"type":"object","required":["embedding_dimensions","is_auto_embedded","name"],"properties":{"description":{"type":"string","maxLength":1000},"embedding_dimensions":{"type":"integer","maximum":2000,"minimum":8},"is_auto_embedded":{"description":"these needs to be pointers so that we can distinguish between false and unset when validating","type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string","maxLength":40,"minLength":3}}},"models.CreateDocumentRequest":{"type":"object","properties":{"content":{"type":"string"},"document_id":{"type":"string","maxLength":100},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true}}},"models.CreateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"}}},"models.CreateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"}}},"models.Document":{"type":"object","properties":{"content":{"type":"string"},"createdAt":{"type":"string"},"deletedAt":{"type":"string"},"documentID":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"isEmbedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updatedAt":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentCollectionResponse":{"type":"object","properties":{"created_at":{"type":"string"},"description":{"type":"string"},"document_count":{"description":"Number of documents in the collection","type":"integer"},"document_embedded_count":{"description":"Number of documents with embeddings","type":"integer"},"embedding_dimensions":{"type":"integer"},"embedding_model_name":{"type":"string"},"is_auto_embedded":{"type":"boolean"},"is_indexed":{"type":"boolean"},"is_normalized":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentResponse":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"document_id":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"is_embedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentSearchPayload":{"type":"object","properties":{"collection_name":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true},"mmr_lambda":{"type":"number"},"text":{"type":"string"},"type":{"$ref":"#/definitions/models.SearchType"}}},"models.GetDocumentListRequest":{"type":"object","properties":{"document_ids":{"type":"array","items":{"type":"string"}},"uuids":{"type":"array","items":{"type":"string"}}}},"models.Memory":{"type":"object","properties":{"messages":{"type":"array","items":{"$ref":"#/definitions/models.Message"}},"metadata":{"type":"object","additionalProperties":true},"summary":{"$ref":"#/definitions/models.Summary"}}},"models.MemorySearchPayload":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"mmr_lambda":{"type":"number"},"text":{"type":"string"},"type":{"$ref":"#/definitions/models.SearchType"}}},"models.MemorySearchResult":{"type":"object","properties":{"dist":{"type":"number"},"embedding":{"type":"array","items":{"type":"number"}},"message":{"$ref":"#/definitions/models.Message"},"metadata":{"type":"object","additionalProperties":true},"summary":{"description":"reserved for future use","allOf":[{"$ref":"#/definitions/models.Summary"}]}}},"models.Message":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"role":{"type":"string"},"token_count":{"type":"integer"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.SearchType":{"type":"string","enum":["similarity","mmr"],"x-enum-varnames":["SearchTypeSimilarity","SearchTypeMMR"]},"models.Session":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"id":{"type":"integer"},"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"updated_at":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"},"uuid":{"type":"string"}}},"models.Summary":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"recent_message_uuid":{"description":"The most recent message UUID that was used to generate this summary","type":"string"},"token_count":{"type":"integer"},"uuid":{"type":"string"}}},"models.UpdateDocumentCollectionRequest":{"type":"object","properties":{"description":{"type":"string","maxLength":1000},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateDocumentListRequest":{"type":"object","required":["uuid"],"properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true},"uuid":{"type":"string"}}},"models.UpdateDocumentRequest":{"type":"object","properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"}}},"models.UpdateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"},"uuid":{"type":"string"}}},"models.User":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"email":{"type":"string"},"first_name":{"type":"string"},"id":{"type":"integer"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"user_id":{"type":"string"},"uuid":{"type":"string"}}}},"securityDefinitions":{"Bearer":{"description":"Type \"Bearer\" followed by a space and JWT token.","type":"apiKey","name":"Authorization","in":"header"}}}` // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ diff --git a/docs/swagger.json b/docs/swagger.json index ef4e6b04..03a13979 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"schemes":["http","https"],"swagger":"2.0","info":{"title":"Zep REST-like API","contact":{},"license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0.html"},"version":"0.x"},"basePath":"/api/v1","paths":{"/api/v1/collection":{"get":{"security":[{"Bearer":[]}],"description":"Returns a list of all DocumentCollections.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a list of DocumentCollections","responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentCollectionResponse"}}}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}":{"get":{"security":[{"Bearer":[]}],"description":"Returns a DocumentCollection if it exists.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentCollectionResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, an error will be returned.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates a new DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, it will be overwritten.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Deletes a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Updates a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document":{"post":{"security":[{"Bearer":[]}],"description":"Creates Documents in a specified DocumentCollection and returns their UUIDs.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Creates Multiple Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be created","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.CreateDocumentRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"string"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchDelete":{"post":{"security":[{"Bearer":[]}],"description":"Deletes specified Documents from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Deletes Documents from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs of the Documents to be deleted","name":"documentUUIDs","in":"body","required":true,"schema":{"type":"array","items":{"type":"string"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchGet":{"post":{"security":[{"Bearer":[]}],"description":"Returns Documents from a DocumentCollection specified by UUID or ID.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Gets Documents from a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs and IDs of the Documents to be fetched","name":"documentRequest","in":"body","required":true,"schema":{"$ref":"#/definitions/models.GetDocumentListRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentResponse"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchUpdate":{"patch":{"security":[{"Bearer":[]}],"description":"Updates Documents in a specified DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Updates Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be updated","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.UpdateDocumentListRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/uuid/{documentUUID}":{"get":{"security":[{"Bearer":[]}],"description":"Returns specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Gets a Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"Delete specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Delete Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be deleted","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Document Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Updates a Document in a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true},{"description":"Document to be updated","name":"document","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/index/create":{"post":{"security":[{"Bearer":[]}],"description":"Creates an index for the specified DocumentCollection to improve query performance.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates an index for a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"boolean","description":"Force index creation, even if there are too few documents to index","name":"force","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/search":{"post":{"security":[{"Bearer":[]}],"description":"Searches Documents in a DocumentCollection based on provided search criteria.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Searches Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"integer","description":"Limit the number of returned documents","name":"limit","in":"query"},{"type":"boolean","description":"Use MMR to rerank the search results. Not Implemented","name":"mmr","in":"query"},{"description":"Search criteria","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.DocumentSearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Document"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions":{"get":{"security":[{"Bearer":[]}],"description":"get all sessions with optional limit and cursor for pagination","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns all sessions","parameters":[{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"type":"integer","description":"Cursor for pagination)","name":"cursor","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateSessionRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}":{"get":{"security":[{"Bearer":[]}],"description":"get session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns a session by ID","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateSessionRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/memory":{"get":{"security":[{"Bearer":[]}],"description":"get memory by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Returns a memory (latest summary and list of messages) for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Last N messages. Overrides memory_window configuration","name":"lastn","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Memory"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Add memory messages to a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Memory messages","name":"memoryMessages","in":"body","required":true,"schema":{"$ref":"#/definitions/models.Memory"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Delete memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/search":{"post":{"security":[{"Bearer":[]}],"description":"search memory messages by session id and query","consumes":["application/json"],"produces":["application/json"],"tags":["search"],"summary":"Search memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"description":"Search query","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.MemorySearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.MemorySearchResult"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user":{"get":{"security":[{"Bearer":[]}],"description":"list all users with pagination","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all users","parameters":[{"type":"integer","description":"Limit","name":"limit","in":"query"},{"type":"integer","description":"Cursor","name":"cursor","in":"query"}],"responses":{"200":{"description":"Successfully retrieved list of users","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.User"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Add a user","parameters":[{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateUserRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}":{"get":{"security":[{"Bearer":[]}],"description":"get user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Returns a user by ID","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Delete a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"update user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Update a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true},{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateUserRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}/sessions":{"get":{"security":[{"Bearer":[]}],"description":"list all sessions for a user by user id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all sessions for a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}}},"definitions":{"apihandlers.APIError":{"type":"object","properties":{"message":{"type":"string"}}},"models.CreateDocumentCollectionRequest":{"type":"object","required":["embedding_dimensions","is_auto_embedded","name"],"properties":{"description":{"type":"string","maxLength":1000},"embedding_dimensions":{"type":"integer","maximum":2000,"minimum":8},"is_auto_embedded":{"description":"these needs to be pointers so that we can distinguish between false and unset when validating","type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string","maxLength":40,"minLength":3}}},"models.CreateDocumentRequest":{"type":"object","properties":{"content":{"type":"string"},"document_id":{"type":"string","maxLength":100},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true}}},"models.CreateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"}}},"models.CreateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"}}},"models.Document":{"type":"object","properties":{"content":{"type":"string"},"createdAt":{"type":"string"},"deletedAt":{"type":"string"},"documentID":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"isEmbedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updatedAt":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentCollectionResponse":{"type":"object","properties":{"created_at":{"type":"string"},"description":{"type":"string"},"document_count":{"description":"Number of documents in the collection","type":"integer"},"document_embedded_count":{"description":"Number of documents with embeddings","type":"integer"},"embedding_dimensions":{"type":"integer"},"embedding_model_name":{"type":"string"},"is_auto_embedded":{"type":"boolean"},"is_indexed":{"type":"boolean"},"is_normalized":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentResponse":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"document_id":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"is_embedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentSearchPayload":{"type":"object","properties":{"collection_name":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true},"text":{"type":"string"}}},"models.GetDocumentListRequest":{"type":"object","properties":{"document_ids":{"type":"array","items":{"type":"string"}},"uuids":{"type":"array","items":{"type":"string"}}}},"models.Memory":{"type":"object","properties":{"messages":{"type":"array","items":{"$ref":"#/definitions/models.Message"}},"metadata":{"type":"object","additionalProperties":true},"summary":{"$ref":"#/definitions/models.Summary"}}},"models.MemorySearchPayload":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"text":{"type":"string"}}},"models.MemorySearchResult":{"type":"object","properties":{"dist":{"type":"number"},"message":{"$ref":"#/definitions/models.Message"},"metadata":{"type":"object","additionalProperties":true},"summary":{"description":"reserved for future use","allOf":[{"$ref":"#/definitions/models.Summary"}]}}},"models.Message":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"role":{"type":"string"},"token_count":{"type":"integer"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.Session":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"id":{"type":"integer"},"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"updated_at":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"},"uuid":{"type":"string"}}},"models.Summary":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"recent_message_uuid":{"description":"The most recent message UUID that was used to generate this summary","type":"string"},"token_count":{"type":"integer"},"uuid":{"type":"string"}}},"models.UpdateDocumentCollectionRequest":{"type":"object","properties":{"description":{"type":"string","maxLength":1000},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateDocumentListRequest":{"type":"object","required":["uuid"],"properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true},"uuid":{"type":"string"}}},"models.UpdateDocumentRequest":{"type":"object","properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"}}},"models.UpdateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"},"uuid":{"type":"string"}}},"models.User":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"email":{"type":"string"},"first_name":{"type":"string"},"id":{"type":"integer"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"user_id":{"type":"string"},"uuid":{"type":"string"}}}},"securityDefinitions":{"Bearer":{"description":"Type \"Bearer\" followed by a space and JWT token.","type":"apiKey","name":"Authorization","in":"header"}}} \ No newline at end of file +{"schemes":["http","https"],"swagger":"2.0","info":{"title":"Zep REST-like API","contact":{},"license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0.html"},"version":"0.x"},"basePath":"/api/v1","paths":{"/api/v1/collection":{"get":{"security":[{"Bearer":[]}],"description":"Returns a list of all DocumentCollections.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a list of DocumentCollections","responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentCollectionResponse"}}}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}":{"get":{"security":[{"Bearer":[]}],"description":"Returns a DocumentCollection if it exists.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Gets a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentCollectionResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, an error will be returned.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates a new DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"If a collection with the same name already exists, it will be overwritten.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Deletes a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Updates a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Document Collection","name":"collection","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentCollectionRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document":{"post":{"security":[{"Bearer":[]}],"description":"Creates Documents in a specified DocumentCollection and returns their UUIDs.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Creates Multiple Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be created","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.CreateDocumentRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"string"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchDelete":{"post":{"security":[{"Bearer":[]}],"description":"Deletes specified Documents from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Deletes Documents from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs of the Documents to be deleted","name":"documentUUIDs","in":"body","required":true,"schema":{"type":"array","items":{"type":"string"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchGet":{"post":{"security":[{"Bearer":[]}],"description":"Returns Documents from a DocumentCollection specified by UUID or ID.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Gets Documents from a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"UUIDs and IDs of the Documents to be fetched","name":"documentRequest","in":"body","required":true,"schema":{"$ref":"#/definitions/models.GetDocumentListRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.DocumentResponse"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/batchUpdate":{"patch":{"security":[{"Bearer":[]}],"description":"Updates Documents in a specified DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Batch Updates Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"description":"Array of Documents to be updated","name":"documents","in":"body","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/models.UpdateDocumentListRequest"}}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/document/uuid/{documentUUID}":{"get":{"security":[{"Bearer":[]}],"description":"Returns specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Gets a Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.DocumentResponse"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"Delete specified Document from a DocumentCollection.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Delete Document from a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be deleted","name":"documentUUID","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Document Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Updates a Document in a DocumentCollection by UUID","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"string","description":"UUID of the Document to be updated","name":"documentUUID","in":"path","required":true},{"description":"Document to be updated","name":"document","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateDocumentRequest"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/index/create":{"post":{"security":[{"Bearer":[]}],"description":"Creates an index for the specified DocumentCollection to improve query performance.","consumes":["application/json"],"produces":["application/json"],"tags":["collection"],"summary":"Creates an index for a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"boolean","description":"Force index creation, even if there are too few documents to index","name":"force","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/collection/{collectionName}/search":{"post":{"security":[{"Bearer":[]}],"description":"Searches Documents in a DocumentCollection based on provided search criteria.","consumes":["application/json"],"produces":["application/json"],"tags":["document"],"summary":"Searches Documents in a DocumentCollection","parameters":[{"type":"string","description":"Name of the Document Collection","name":"collectionName","in":"path","required":true},{"type":"integer","description":"Limit the number of returned documents","name":"limit","in":"query"},{"description":"Search criteria","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.DocumentSearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Document"}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"401":{"description":"Unauthorized","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions":{"get":{"security":[{"Bearer":[]}],"description":"get all sessions with optional limit and cursor for pagination","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns all sessions","parameters":[{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"type":"integer","description":"Cursor for pagination)","name":"cursor","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateSessionRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}":{"get":{"security":[{"Bearer":[]}],"description":"get session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Returns a session by ID","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"add session by id","consumes":["application/json"],"produces":["application/json"],"tags":["session"],"summary":"Add a session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Session","name":"session","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateSessionRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.Session"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/memory":{"get":{"security":[{"Bearer":[]}],"description":"get memory by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Returns a memory (latest summary and list of messages) for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Last N messages. Overrides memory_window configuration","name":"lastn","in":"query"}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Memory"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Add memory messages to a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"description":"Memory messages","name":"memoryMessages","in":"body","required":true,"schema":{"$ref":"#/definitions/models.Memory"}}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete memory messages by session id","consumes":["application/json"],"produces":["application/json"],"tags":["memory"],"summary":"Delete memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/sessions/{sessionId}/search":{"post":{"security":[{"Bearer":[]}],"description":"search memory messages by session id and query","consumes":["application/json"],"produces":["application/json"],"tags":["search"],"summary":"Search memory messages for a given session","parameters":[{"type":"string","description":"Session ID","name":"sessionId","in":"path","required":true},{"type":"integer","description":"Limit the number of results returned","name":"limit","in":"query"},{"description":"Search query","name":"searchPayload","in":"body","required":true,"schema":{"$ref":"#/definitions/models.MemorySearchPayload"}}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.MemorySearchResult"}}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user":{"get":{"security":[{"Bearer":[]}],"description":"list all users with pagination","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all users","parameters":[{"type":"integer","description":"Limit","name":"limit","in":"query"},{"type":"integer","description":"Cursor","name":"cursor","in":"query"}],"responses":{"200":{"description":"Successfully retrieved list of users","schema":{"type":"array","items":{"type":"array","items":{"$ref":"#/definitions/models.User"}}}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"post":{"security":[{"Bearer":[]}],"description":"add user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Add a user","parameters":[{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.CreateUserRequest"}}],"responses":{"201":{"description":"Created","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}":{"get":{"security":[{"Bearer":[]}],"description":"get user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Returns a user by ID","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"delete":{"security":[{"Bearer":[]}],"description":"delete user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Delete a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"string"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}},"patch":{"security":[{"Bearer":[]}],"description":"update user by id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"Update a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true},{"description":"User","name":"user","in":"body","required":true,"schema":{"$ref":"#/definitions/models.UpdateUserRequest"}}],"responses":{"200":{"description":"OK","schema":{"$ref":"#/definitions/models.User"}},"400":{"description":"Bad Request","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"404":{"description":"Not Found","schema":{"$ref":"#/definitions/apihandlers.APIError"}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}},"/api/v1/user/{userId}/sessions":{"get":{"security":[{"Bearer":[]}],"description":"list all sessions for a user by user id","consumes":["application/json"],"produces":["application/json"],"tags":["user"],"summary":"List all sessions for a user","parameters":[{"type":"string","description":"User ID","name":"userId","in":"path","required":true}],"responses":{"200":{"description":"OK","schema":{"type":"array","items":{"$ref":"#/definitions/models.Session"}}},"500":{"description":"Internal Server Error","schema":{"$ref":"#/definitions/apihandlers.APIError"}}}}}},"definitions":{"apihandlers.APIError":{"type":"object","properties":{"message":{"type":"string"}}},"models.CreateDocumentCollectionRequest":{"type":"object","required":["embedding_dimensions","is_auto_embedded","name"],"properties":{"description":{"type":"string","maxLength":1000},"embedding_dimensions":{"type":"integer","maximum":2000,"minimum":8},"is_auto_embedded":{"description":"these needs to be pointers so that we can distinguish between false and unset when validating","type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string","maxLength":40,"minLength":3}}},"models.CreateDocumentRequest":{"type":"object","properties":{"content":{"type":"string"},"document_id":{"type":"string","maxLength":100},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true}}},"models.CreateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"}}},"models.CreateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"}}},"models.Document":{"type":"object","properties":{"content":{"type":"string"},"createdAt":{"type":"string"},"deletedAt":{"type":"string"},"documentID":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"isEmbedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updatedAt":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentCollectionResponse":{"type":"object","properties":{"created_at":{"type":"string"},"description":{"type":"string"},"document_count":{"description":"Number of documents in the collection","type":"integer"},"document_embedded_count":{"description":"Number of documents with embeddings","type":"integer"},"embedding_dimensions":{"type":"integer"},"embedding_model_name":{"type":"string"},"is_auto_embedded":{"type":"boolean"},"is_indexed":{"type":"boolean"},"is_normalized":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"name":{"type":"string"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentResponse":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"document_id":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"is_embedded":{"type":"boolean"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.DocumentSearchPayload":{"type":"object","properties":{"collection_name":{"type":"string"},"embedding":{"type":"array","items":{"type":"number"}},"metadata":{"type":"object","additionalProperties":true},"mmr_lambda":{"type":"number"},"text":{"type":"string"},"type":{"$ref":"#/definitions/models.SearchType"}}},"models.GetDocumentListRequest":{"type":"object","properties":{"document_ids":{"type":"array","items":{"type":"string"}},"uuids":{"type":"array","items":{"type":"string"}}}},"models.Memory":{"type":"object","properties":{"messages":{"type":"array","items":{"$ref":"#/definitions/models.Message"}},"metadata":{"type":"object","additionalProperties":true},"summary":{"$ref":"#/definitions/models.Summary"}}},"models.MemorySearchPayload":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"mmr_lambda":{"type":"number"},"text":{"type":"string"},"type":{"$ref":"#/definitions/models.SearchType"}}},"models.MemorySearchResult":{"type":"object","properties":{"dist":{"type":"number"},"embedding":{"type":"array","items":{"type":"number"}},"message":{"$ref":"#/definitions/models.Message"},"metadata":{"type":"object","additionalProperties":true},"summary":{"description":"reserved for future use","allOf":[{"$ref":"#/definitions/models.Summary"}]}}},"models.Message":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"role":{"type":"string"},"token_count":{"type":"integer"},"updated_at":{"type":"string"},"uuid":{"type":"string"}}},"models.SearchType":{"type":"string","enum":["similarity","mmr"],"x-enum-varnames":["SearchTypeSimilarity","SearchTypeMMR"]},"models.Session":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"id":{"type":"integer"},"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"},"updated_at":{"type":"string"},"user_id":{"description":"Must be a pointer to allow for null values","type":"string"},"uuid":{"type":"string"}}},"models.Summary":{"type":"object","properties":{"content":{"type":"string"},"created_at":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"recent_message_uuid":{"description":"The most recent message UUID that was used to generate this summary","type":"string"},"token_count":{"type":"integer"},"uuid":{"type":"string"}}},"models.UpdateDocumentCollectionRequest":{"type":"object","properties":{"description":{"type":"string","maxLength":1000},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateDocumentListRequest":{"type":"object","required":["uuid"],"properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true},"uuid":{"type":"string"}}},"models.UpdateDocumentRequest":{"type":"object","properties":{"document_id":{"type":"string","maxLength":40},"metadata":{"type":"object","additionalProperties":true}}},"models.UpdateSessionRequest":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":true},"session_id":{"type":"string"}}},"models.UpdateUserRequest":{"type":"object","properties":{"email":{"type":"string"},"first_name":{"type":"string"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"user_id":{"type":"string"},"uuid":{"type":"string"}}},"models.User":{"type":"object","properties":{"created_at":{"type":"string"},"deleted_at":{"type":"string"},"email":{"type":"string"},"first_name":{"type":"string"},"id":{"type":"integer"},"last_name":{"type":"string"},"metadata":{"type":"object","additionalProperties":true},"updated_at":{"type":"string"},"user_id":{"type":"string"},"uuid":{"type":"string"}}}},"securityDefinitions":{"Bearer":{"description":"Type \"Bearer\" followed by a space and JWT token.","type":"apiKey","name":"Authorization","in":"header"}}} \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 7523f8e0..725effe0 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -159,8 +159,12 @@ definitions: metadata: additionalProperties: true type: object + mmr_lambda: + type: number text: type: string + type: + $ref: '#/definitions/models.SearchType' type: object models.GetDocumentListRequest: properties: @@ -190,13 +194,21 @@ definitions: metadata: additionalProperties: true type: object + mmr_lambda: + type: number text: type: string + type: + $ref: '#/definitions/models.SearchType' type: object models.MemorySearchResult: properties: dist: type: number + embedding: + items: + type: number + type: array message: $ref: '#/definitions/models.Message' metadata: @@ -225,6 +237,14 @@ definitions: uuid: type: string type: object + models.SearchType: + enum: + - similarity + - mmr + type: string + x-enum-varnames: + - SearchTypeSimilarity + - SearchTypeMMR models.Session: properties: created_at: @@ -910,10 +930,6 @@ paths: in: query name: limit type: integer - - description: Use MMR to rerank the search results. Not Implemented - in: query - name: mmr - type: boolean - description: Search criteria in: body name: searchPayload diff --git a/pkg/server/routes.go b/pkg/server/routes.go index 621b5b80..01113213 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -37,16 +37,16 @@ func Create(appState *models.AppState) *http.Server { } } -// @title Zep REST-like API -// @version 0.x -// @license.name Apache 2.0 -// @license.url http://www.apache.org/licenses/LICENSE-2.0.html -// @BasePath /api/v1 -// @schemes http https -// @securityDefinitions.apikey Bearer -// @in header -// @name Authorization -// @description Type "Bearer" followed by a space and JWT token. +// @title Zep REST-like API +// @version 0.x +// @license.name Apache 2.0 +// @license.url http://www.apache.org/licenses/LICENSE-2.0.html +// @BasePath /api/v1 +// @schemes http https +// @securityDefinitions.apikey Bearer +// @in header +// @name Authorization +// @description Type "Bearer" followed by a space and JWT token. func setupRouter(appState *models.AppState) *chi.Mux { maxRequestSize := appState.Config.Server.MaxRequestSize if maxRequestSize == 0 { From 64163a097131a4746b7d2fa02b8dcc16a1f222c1 Mon Sep 17 00:00:00 2001 From: Daniel Chalef <131175+danielchalef@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:43:26 -0700 Subject: [PATCH 4/4] mmr for document search --- pkg/models/document.go | 2 +- pkg/server/document_routes_test.go | 69 +++++++++++------ pkg/store/postgres/document_search.go | 57 +++++++++++--- pkg/store/postgres/document_search_test.go | 86 ++++++++++++++++------ pkg/store/postgres/search_memory.go | 2 - pkg/store/postgres/search_utils.go | 3 + 6 files changed, 160 insertions(+), 59 deletions(-) diff --git a/pkg/models/document.go b/pkg/models/document.go index 5d6a7c41..85f101a8 100644 --- a/pkg/models/document.go +++ b/pkg/models/document.go @@ -83,7 +83,7 @@ type Document struct { Embedding []float32 `bun:"type:vector,nullzero" json:"embedding,omitempty"` } -type SearchDocumentQuery struct { +type SearchDocumentResult struct { *Document Score float64 `json:"score" bun:"score"` } diff --git a/pkg/server/document_routes_test.go b/pkg/server/document_routes_test.go index e2099efa..5d4c8d67 100644 --- a/pkg/server/document_routes_test.go +++ b/pkg/server/document_routes_test.go @@ -345,34 +345,59 @@ func TestSearchDocumentsHandler(t *testing.T) { }, Embedding: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, -0.1, -0.2, -0.3, -0.4}, }, + { + DocumentBase: models.DocumentBase{ + DocumentID: "doc3", + Content: "This is a third test document", + Metadata: map[string]interface{}{ + "key": "value", + }, + }, + Embedding: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, -0.1, -0.2, -0.3, -0.5}, + }, } _, err = appState.DocumentStore.CreateDocuments(testCtx, collectionName, docs) assert.NoError(t, err) - q := models.DocumentSearchPayload{ - CollectionName: collectionName, - Embedding: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, -0.1, -0.2, -0.3, -0.4}, - Metadata: map[string]interface{}{ - "where": map[string]interface{}{"jsonpath": "$[*] ? (@.key == 'value')"}, - }, - } - p, err := json.Marshal(q) - assert.NoError(t, err) + // Test cases for different search types + testCases := []models.SearchType{models.SearchTypeSimilarity, models.SearchTypeMMR} - // Create a new HTTP request - req, err := http.NewRequest( - "POST", - testServer.URL+"/api/v1/collection/"+collectionName+"/search", - bytes.NewBuffer(p), - ) - assert.NoError(t, err) + for _, searchType := range testCases { + q := models.DocumentSearchPayload{ + CollectionName: collectionName, + Embedding: []float32{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, -0.1, -0.2, -0.3, -0.4}, + Metadata: map[string]interface{}{ + "where": map[string]interface{}{"jsonpath": "$[*] ? (@.key == 'value')"}, + }, + Type: searchType, + } + p, err := json.Marshal(q) + assert.NoError(t, err) - // Send the request - client := &http.Client{} - resp, err := client.Do(req) - assert.NoError(t, err) + limit := "2" - // Check the status code - assert.Equal(t, http.StatusOK, resp.StatusCode) + // Create a new HTTP request + req, err := http.NewRequest( + "POST", + testServer.URL+"/api/v1/collection/"+collectionName+"/search?limit="+limit, + bytes.NewBuffer(p), + ) + assert.NoError(t, err) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(t, err) + + // Check the status code + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // Check the response body + var searchResults models.DocumentSearchResultPage + err = json.NewDecoder(resp.Body).Decode(&searchResults) + assert.NoError(t, err) + + assert.Equal(t, 2, len(searchResults.Results)) + } } diff --git a/pkg/store/postgres/document_search.go b/pkg/store/postgres/document_search.go index 07efba4a..43cf25de 100644 --- a/pkg/store/postgres/document_search.go +++ b/pkg/store/postgres/document_search.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/getzep/zep/pkg/llms" + "github.com/getzep/zep/pkg/search" "github.com/getzep/zep/pkg/store" "github.com/pgvector/pgvector-go" "github.com/uptrace/bun" @@ -49,11 +50,10 @@ type documentSearchOperation struct { collection *models.DocumentCollection queryVector []float32 limit int - withMMR bool } func (dso *documentSearchOperation) Execute() (*models.DocumentSearchResultPage, error) { - var results []models.SearchDocumentQuery + var results []models.SearchDocumentResult var count int var err error @@ -91,6 +91,13 @@ func (dso *documentSearchOperation) Execute() (*models.DocumentSearchResultPage, return nil, fmt.Errorf("error executing search: %w", err) } + if dso.searchPayload.Type == models.SearchTypeMMR { + results, err = dso.reRankMMR(results) + if err != nil { + return nil, fmt.Errorf("error reranking results: %w", err) + } + } + resultPage := &models.DocumentSearchResultPage{ Results: searchResultsFromSearchQueries(results), QueryVector: dso.queryVector, @@ -100,10 +107,42 @@ func (dso *documentSearchOperation) Execute() (*models.DocumentSearchResultPage, return resultPage, nil } +// reRankMMR reranks the results using the MMR algorithm. +func (dso *documentSearchOperation) reRankMMR( + results []models.SearchDocumentResult, +) ([]models.SearchDocumentResult, error) { + lambda := dso.searchPayload.MMRLambda + if lambda == 0 { + lambda = DefaultMMRLambda + } + + k := dso.limit + if k == 0 { + k = DefaultDocumentSearchLimit + } + + resultVectors := make([][]float32, len(results)) + for i := range results { + resultVectors[i] = results[i].Embedding + } + + rankedIndices, err := search.MaximalMarginalRelevance(dso.queryVector, resultVectors, lambda, k) + if err != nil { + return nil, fmt.Errorf("error reranking results: %w", err) + } + + rankedResults := make([]models.SearchDocumentResult, len(rankedIndices)) + for i := range rankedIndices { + rankedResults[i] = results[rankedIndices[i]] + } + + return rankedResults, nil +} + // execQuery executes the query and scans the results into the provided results slice. It accepts a bun DB or Tx. func (dso *documentSearchOperation) execQuery( db bun.IDB, - results *[]models.SearchDocumentQuery, + results *[]models.SearchDocumentResult, ) (int, error) { query, err := dso.buildQuery(db) if err != nil { @@ -120,15 +159,11 @@ func (dso *documentSearchOperation) execQuery( count := len(*results) - if count == 0 { - return 0, models.NewNotFoundError("no results found") - } - return count, nil } func (dso *documentSearchOperation) buildQuery(db bun.IDB) (*bun.SelectQuery, error) { - m := &[]models.SearchDocumentQuery{} + m := &[]models.SearchDocumentResult{} query := db.NewSelect().Model(m). ModelTableExpr("?", bun.Ident(dso.collection.TableName)). Column("*"). @@ -165,8 +200,8 @@ func (dso *documentSearchOperation) buildQuery(db bun.IDB) (*bun.SelectQuery, er // If we're using MMR, we need to add a limit of 2x the requested limit to allow for the MMR // algorithm to rerank and filter out results. limit := dso.limit - if dso.withMMR { - limit *= 2 + if dso.searchPayload.Type == models.SearchTypeMMR { + limit *= DefaultMMRMultiplier } query = query.Limit(limit) @@ -223,7 +258,7 @@ func (dso *documentSearchOperation) applyDocsMetadataFilter( return query, nil } -func searchResultsFromSearchQueries(s []models.SearchDocumentQuery) []models.DocumentSearchResult { +func searchResultsFromSearchQueries(s []models.SearchDocumentResult) []models.DocumentSearchResult { result := make([]models.DocumentSearchResult, len(s)) for i := range s { diff --git a/pkg/store/postgres/document_search_test.go b/pkg/store/postgres/document_search_test.go index 5777e9e6..40fa7101 100644 --- a/pkg/store/postgres/document_search_test.go +++ b/pkg/store/postgres/document_search_test.go @@ -15,7 +15,6 @@ import ( // TODO: Test non-happy paths // End to end with local embedding -// won't pass if NLP service is running via docker-compose func TestDocumentSearchWithIndexEndToEnd(t *testing.T) { gofakeit.Seed(0) ctx, done := context.WithCancel(testCtx) @@ -43,28 +42,6 @@ func TestDocumentSearchWithIndexEndToEnd(t *testing.T) { 500, false, true, 1536) assert.NoError(t, err) - // TODO: Test without HNSW - // collection := docCollection.collection.DocumentCollection - // // create index - // vci, err := NewVectorColIndex( - // ctx, - // appState, - // collection, - // ) - // assert.NoError(t, err) - - // err = vci.CreateIndex(context.Background(), true) - // assert.NoError(t, err) - - // pollIndexCreation(documentStore, collectionName, ctx, t) - - // // Set Collection's IsIndexed flag to true - // col, err := documentStore.GetCollection(ctx, vci.Collection.Name) - // assert.NoError(t, err) - // assert.Equal(t, true, col.IsIndexed) - // assert.True(t, col.ProbeCount > 0) - // assert.True(t, col.ListCount > 0) - limit := 5 searchPayload := &models.DocumentSearchPayload{ Text: gofakeit.HipsterParagraph(2, 2, 12, " "), @@ -96,3 +73,66 @@ func TestDocumentSearchWithIndexEndToEnd(t *testing.T) { done() } + +func TestReRankMMR(t *testing.T) { + // Initialize a documentSearchOperation with a searchPayload of type MMR + dso := &documentSearchOperation{ + searchPayload: &models.DocumentSearchPayload{ + Type: models.SearchTypeMMR, + MMRLambda: 0.5, + }, + queryVector: []float32{0.1, 0.2, 0.3}, + limit: 2, + } + + // Create a slice of SearchDocumentResult + results := []models.SearchDocumentResult{ + { + Document: &models.Document{ + DocumentBase: models.DocumentBase{ + DocumentID: "doc1", + }, + Embedding: []float32{0.1, 0.2, 0.3}, + }, + Score: 1.0, + }, + { + Document: &models.Document{ + DocumentBase: models.DocumentBase{ + DocumentID: "doc2", + }, + Embedding: []float32{0.4, 0.5, 0.6}, + }, + Score: 0.4, + }, + { + Document: &models.Document{ + DocumentBase: models.DocumentBase{ + DocumentID: "doc3", + }, + Embedding: []float32{0.7, 0.8, 0.9}, + }, + Score: 0.2, + }, + { + Document: &models.Document{ + DocumentBase: models.DocumentBase{ + DocumentID: "doc4", + }, + Embedding: []float32{0.1, 0.2, 0.4}, + }, + Score: 0.8, + }, + } + + // Call reRankMMR method + rankedResults, err := dso.reRankMMR(results) + + // Assert no error was returned + assert.NoError(t, err) + + // Assert that the results have been reranked correctly + assert.Equal(t, 2, len(rankedResults)) + assert.Equal(t, "doc1", rankedResults[0].Document.DocumentID) + assert.Equal(t, "doc2", rankedResults[1].Document.DocumentID) +} diff --git a/pkg/store/postgres/search_memory.go b/pkg/store/postgres/search_memory.go index 01710ad1..fdce96a1 100644 --- a/pkg/store/postgres/search_memory.go +++ b/pkg/store/postgres/search_memory.go @@ -15,8 +15,6 @@ import ( ) const DefaultMemorySearchLimit = 10 -const DefaultMMRMultiplier = 2 -const DefaultMMRLambda = 0.5 type JSONQuery struct { JSONPath string `json:"jsonpath"` diff --git a/pkg/store/postgres/search_utils.go b/pkg/store/postgres/search_utils.go index 2ec1aafa..5a984f8a 100644 --- a/pkg/store/postgres/search_utils.go +++ b/pkg/store/postgres/search_utils.go @@ -7,6 +7,9 @@ import ( "github.com/uptrace/bun" ) +const DefaultMMRMultiplier = 2 +const DefaultMMRLambda = 0.5 + // parseJSONQuery recursively parses a JSONQuery and returns a bun.QueryBuilder. // TODO: fix the addition of extraneous parentheses in the query func parseJSONQuery(