diff --git a/channels/log_entry.go b/channels/log_entry.go index 4e9112c03e..cf51a446b6 100644 --- a/channels/log_entry.go +++ b/channels/log_entry.go @@ -14,6 +14,7 @@ package channels import ( "fmt" + "strconv" "time" "github.com/couchbase/sync_gateway/base" @@ -126,3 +127,9 @@ func (rv *RevAndVersion) UnmarshalJSON(data []byte) error { return fmt.Errorf("unrecognized JSON format for RevAndVersion: %s", data) } } + +// CV returns ver@src in big endian format 1@cbl for CBL format. +func (rv RevAndVersion) CV() string { + // this should match db.Version.String() + return strconv.FormatUint(base.HexCasToUint64(rv.CurrentVersion), 16) + "@" + rv.CurrentSource +} diff --git a/db/crud.go b/db/crud.go index 3d86ce3b1a..54ce013337 100644 --- a/db/crud.go +++ b/db/crud.go @@ -287,12 +287,25 @@ func (db *DatabaseCollectionWithUser) Get1xRevBody(ctx context.Context, docid, r maxHistory = math.MaxInt32 } - return db.Get1xRevBodyWithHistory(ctx, docid, revid, maxHistory, nil, attachmentsSince, false) + return db.Get1xRevBodyWithHistory(ctx, docid, revid, Get1xRevBodyOptions{ + MaxHistory: maxHistory, + HistoryFrom: nil, + AttachmentsSince: attachmentsSince, + ShowExp: false, + }) +} + +type Get1xRevBodyOptions struct { + MaxHistory int + HistoryFrom []string + AttachmentsSince []string + ShowExp bool + ShowCV bool } // Retrieves rev with request history specified as collection of revids (historyFrom) -func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revid string, maxHistory int, historyFrom []string, attachmentsSince []string, showExp bool) (Body, error) { - rev, err := db.getRev(ctx, docid, revid, maxHistory, historyFrom) +func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revtreeid string, opts Get1xRevBodyOptions) (Body, error) { + rev, err := db.getRev(ctx, docid, revtreeid, opts.MaxHistory, opts.HistoryFrom) if err != nil { return nil, err } @@ -300,14 +313,14 @@ func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Contex // RequestedHistory is the _revisions returned in the body. Avoids mutating revision.History, in case it's needed // during attachment processing below requestedHistory := rev.History - if maxHistory == 0 { + if opts.MaxHistory == 0 { requestedHistory = nil } if requestedHistory != nil { - _, requestedHistory = trimEncodedRevisionsToAncestor(ctx, requestedHistory, historyFrom, maxHistory) + _, requestedHistory = trimEncodedRevisionsToAncestor(ctx, requestedHistory, opts.HistoryFrom, opts.MaxHistory) } - return rev.Mutable1xBody(ctx, db, requestedHistory, attachmentsSince, showExp) + return rev.Mutable1xBody(ctx, db, requestedHistory, opts.AttachmentsSince, opts.ShowExp, opts.ShowCV) } // Underlying revision retrieval used by Get1xRevBody, Get1xRevBodyWithHistory, GetRevCopy. diff --git a/db/database.go b/db/database.go index a62462b698..747b4e71ed 100644 --- a/db/database.go +++ b/db/database.go @@ -952,6 +952,7 @@ type IDRevAndSequence struct { DocID string RevID string Sequence uint64 + CV string } // The ForEachDocID options for limiting query results @@ -987,6 +988,7 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal var found bool var docid, revid string var seq uint64 + var cv string var channels []string if c.useViews() { var viewRow AllDocsViewQueryRow @@ -994,6 +996,7 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal if found { docid = viewRow.Key revid = viewRow.Value.RevID.RevTreeID + cv = viewRow.Value.RevID.CV() seq = viewRow.Value.Sequence channels = viewRow.Value.Channels } @@ -1002,6 +1005,7 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal if found { docid = queryRow.Id revid = queryRow.RevID.RevTreeID + cv = queryRow.RevID.CV() seq = queryRow.Sequence channels = make([]string, 0) // Query returns all channels, but we only want to return active channels @@ -1016,7 +1020,7 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal break } - if ok, err := callback(IDRevAndSequence{docid, revid, seq}, channels); ok { + if ok, err := callback(IDRevAndSequence{DocID: docid, RevID: revid, Sequence: seq, CV: cv}, channels); ok { count++ } else if err != nil { return err diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 5639328db5..bb11e0ff40 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -139,7 +139,7 @@ func ParseVersion(versionString string) (version Version, err error) { return version, nil } -// String returns a version/sourceID pair in CBL string format +// String returns a version/sourceID pair in CBL string format. This does not match the format serialized on CBS, which will be in 0x0 format. func (v Version) String() string { return strconv.FormatUint(v.Value, 16) + "@" + v.SourceID } diff --git a/db/revision_cache_interface.go b/db/revision_cache_interface.go index 5147929abf..166c415fb8 100644 --- a/db/revision_cache_interface.go +++ b/db/revision_cache_interface.go @@ -262,7 +262,7 @@ func (rev *DocumentRevision) Inject1xBodyProperties(ctx context.Context, db *Dat // Mutable1xBody returns a copy of the given document revision as a 1.x style body (with special properties) // Callers are free to modify this body without affecting the document revision. -func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseCollectionWithUser, requestedHistory Revisions, attachmentsSince []string, showExp bool) (b Body, err error) { +func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseCollectionWithUser, requestedHistory Revisions, attachmentsSince []string, showExp bool, showCV bool) (b Body, err error) { b, err = rev.Body() if err != nil { return nil, err @@ -283,6 +283,10 @@ func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseColl b[BodyExpiry] = rev.Expiry.Format(time.RFC3339) } + if showCV && rev.CV != nil { + b["_cv"] = rev.CV.String() + } + if rev.Deleted { b[BodyDeleted] = true } diff --git a/rest/access_test.go b/rest/access_test.go index 4d5a9136f8..ec93172ff9 100644 --- a/rest/access_test.go +++ b/rest/access_test.go @@ -25,6 +25,12 @@ import ( "github.com/stretchr/testify/require" ) +type allDocsResponse struct { + TotalRows int `json:"total_rows"` + Offset int `json:"offset"` + Rows []allDocsRow `json:"rows"` +} + func TestPublicChanGuestAccess(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{ @@ -70,17 +76,6 @@ func TestStarAccess(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyChanges) - type allDocsRow struct { - ID string `json:"id"` - Key string `json:"key"` - Value struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } `json:"value"` - Doc db.Body `json:"doc,omitempty"` - Error string `json:"error"` - } var allDocsResult struct { TotalRows int `json:"total_rows"` Offset int `json:"offset"` @@ -552,23 +547,6 @@ func TestAllDocsAccessControl(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{SyncFn: channels.DocChannelsSyncFunction}) defer rt.Close() - type allDocsRow struct { - ID string `json:"id"` - Key string `json:"key"` - Value struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } `json:"value"` - Doc db.Body `json:"doc,omitempty"` - Error string `json:"error"` - } - type allDocsResponse struct { - TotalRows int `json:"total_rows"` - Offset int `json:"offset"` - Rows []allDocsRow `json:"rows"` - } - // Create some docs: a := auth.NewAuthenticator(rt.MetadataStore(), nil, rt.GetDatabase().AuthenticatorOptions(rt.Context())) a.Collections = rt.GetDatabase().CollectionNames @@ -708,13 +686,13 @@ func TestAllDocsAccessControl(t *testing.T) { assert.Equal(t, []string{"Cinemax"}, allDocsResult.Rows[0].Value.Channels) assert.Equal(t, "doc1", allDocsResult.Rows[1].Key) assert.Equal(t, "forbidden", allDocsResult.Rows[1].Error) - assert.Equal(t, "", allDocsResult.Rows[1].Value.Rev) + assert.Nil(t, allDocsResult.Rows[1].Value) assert.Equal(t, "doc3", allDocsResult.Rows[2].ID) assert.Equal(t, []string{"Cinemax"}, allDocsResult.Rows[2].Value.Channels) assert.Equal(t, "1-20912648f85f2bbabefb0993ddd37b41", allDocsResult.Rows[2].Value.Rev) assert.Equal(t, "b0gus", allDocsResult.Rows[3].Key) assert.Equal(t, "not_found", allDocsResult.Rows[3].Error) - assert.Equal(t, "", allDocsResult.Rows[3].Value.Rev) + assert.Nil(t, allDocsResult.Rows[3].Value) // Check GET to _all_docs with keys parameter: response = rt.SendUserRequest(http.MethodGet, "/{{.keyspace}}/_all_docs?channels=true&keys=%5B%22doc4%22%2C%22doc1%22%2C%22doc3%22%2C%22b0gus%22%5D", "", "alice") @@ -1178,3 +1156,43 @@ func TestPublicChannel(t *testing.T) { response = rt.SendUserRequest("GET", "/{{.keyspace}}/privateDoc", "", "user1") RequireStatus(t, response, 403) } + +func TestAllDocsCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + const docID = "foo" + docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + + testCases := []struct { + name string + url string + output string + }{ + { + name: "no query string", + url: "/{{.keyspace}}/_all_docs", + output: fmt.Sprintf(`{ + "total_rows": 1, + "update_seq": 1, + "rows": [{"key": "%s", "id": "%s", "value": {"rev": "%s"}}] + }`, docID, docID, docVersion.RevTreeID), + }, + { + name: "cvs=true", + url: "/{{.keyspace}}/_all_docs?show_cv=true", + output: fmt.Sprintf(`{ + "total_rows": 1, + "update_seq": 1, + "rows": [{"key": "%s", "id": "%s", "value": {"rev": "%s", "cv": "%s"}}] + }`, docID, docID, docVersion.RevTreeID, docVersion.CV.String()), + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequest(http.MethodGet, testCase.url, "") + RequireStatus(t, response, http.StatusOK) + require.JSONEq(t, testCase.output, response.Body.String()) + }) + } +} diff --git a/rest/api_test.go b/rest/api_test.go index bc554c10fe..cc420c5dfe 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -2717,7 +2717,7 @@ func TestNullDocHandlingForMutable1xBody(t *testing.T) { documentRev := db.DocumentRevision{DocID: "doc1", BodyBytes: []byte("null")} - body, err := documentRev.Mutable1xBody(ctx, collection, nil, nil, false) + body, err := documentRev.Mutable1xBody(ctx, collection, nil, nil, false, false) require.Error(t, err) require.Nil(t, body) assert.Contains(t, err.Error(), "null doc body for doc") diff --git a/rest/bulk_api.go b/rest/bulk_api.go index 6c4e50b4c4..c97e4fed41 100644 --- a/rest/bulk_api.go +++ b/rest/bulk_api.go @@ -25,6 +25,25 @@ import ( "github.com/couchbase/sync_gateway/db" ) +// allDocsRowValue is a struct that represents possible values returned in a document from /ks/_all_docs endpoint +type allDocsRowValue struct { + Rev string `json:"rev"` + CV string `json:"cv,omitempty"` + Channels []string `json:"channels,omitempty"` + Access map[string]base.Set `json:"access,omitempty"` // for admins only +} + +// allDocsRow is a struct that represents a linefrom /ks/_all_docs endpoint +type allDocsRow struct { + Key string `json:"key"` + ID string `json:"id,omitempty"` + Value *allDocsRowValue `json:"value,omitempty"` + Doc json.RawMessage `json:"doc,omitempty"` + UpdateSeq uint64 `json:"update_seq,omitempty"` + Error string `json:"error,omitempty"` + Status int `json:"status,omitempty"` +} + // HTTP handler for _all_docs func (h *handler) handleAllDocs() error { // http://wiki.apache.org/couchdb/HTTP_Bulk_Document_API @@ -32,6 +51,7 @@ func (h *handler) handleAllDocs() error { includeChannels := h.getBoolQuery("channels") includeAccess := h.getBoolQuery("access") && h.user == nil includeRevs := h.getBoolQuery("revs") + includeCVs := h.getBoolQuery("show_cv") includeSeqs := h.getBoolQuery("update_seq") // Get the doc IDs if this is a POST request: @@ -99,21 +119,6 @@ func (h *handler) handleAllDocs() error { return result } - type allDocsRowValue struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } - type allDocsRow struct { - Key string `json:"key"` - ID string `json:"id,omitempty"` - Value *allDocsRowValue `json:"value,omitempty"` - Doc json.RawMessage `json:"doc,omitempty"` - UpdateSeq uint64 `json:"update_seq,omitempty"` - Error string `json:"error,omitempty"` - Status int `json:"status,omitempty"` - } - // Subroutine that creates a response row for a document: totalRows := 0 createRow := func(doc db.IDRevAndSequence, channels []string) *allDocsRow { @@ -169,6 +174,9 @@ func (h *handler) handleAllDocs() error { if includeChannels { row.Value.Channels = channels } + if includeCVs { + row.Value.CV = doc.CV + } return row } @@ -220,7 +228,8 @@ func (h *handler) handleAllDocs() error { if explicitDocIDs != nil { count := uint64(0) for _, docID := range explicitDocIDs { - _, _ = writeDoc(db.IDRevAndSequence{DocID: docID, RevID: "", Sequence: 0}, nil) + // no revtreeid or cv if explicitDocIDs are specified + _, _ = writeDoc(db.IDRevAndSequence{DocID: docID, RevID: "", Sequence: 0, CV: ""}, nil) count++ if options.Limit > 0 && count == options.Limit { break @@ -364,6 +373,7 @@ func (h *handler) handleBulkGet() error { includeAttachments := h.getBoolQuery("attachments") showExp := h.getBoolQuery("show_exp") + showCV := h.getBoolQuery("show_cv") showRevs := h.getBoolQuery("revs") globalRevsLimit := int(h.getIntQuery("revs_limit", math.MaxInt32)) @@ -440,7 +450,12 @@ func (h *handler) handleBulkGet() error { } if err == nil { - body, err = h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, docRevsLimit, revsFrom, attsSince, showExp) + body, err = h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: docRevsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attsSince, + ShowExp: showExp, + ShowCV: showCV}) } if err != nil { diff --git a/rest/doc_api.go b/rest/doc_api.go index b7182b313e..074a069153 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -30,6 +30,7 @@ func (h *handler) handleGetDoc() error { revid := h.getQuery("rev") openRevs := h.getQuery("open_revs") showExp := h.getBoolQuery("show_exp") + showCV := h.getBoolQuery("show_cv") if replicator2, _ := h.getOptBoolQuery("replicator2", false); replicator2 { return h.handleGetDocReplicator2(docid, revid) @@ -68,7 +69,12 @@ func (h *handler) handleGetDoc() error { if openRevs == "" { // Single-revision GET: - value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { if err == base.ErrImportCancelledPurged { base.DebugfCtx(h.ctx(), base.KeyImport, fmt.Sprintf("Import cancelled as document %v is purged", base.UD(docid))) @@ -130,7 +136,12 @@ func (h *handler) handleGetDoc() error { if h.requestAccepts("multipart/") { err := h.writeMultipart("mixed", func(writer *multipart.Writer) error { for _, revid := range revids { - revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { revBody = db.Body{"missing": revid} // TODO: More specific error } @@ -152,7 +163,12 @@ func (h *handler) handleGetDoc() error { _, _ = h.response.Write([]byte(`[` + "\n")) separator := []byte(``) for _, revid := range revids { - revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { revBody = db.Body{"missing": revid} // TODO: More specific error } else { diff --git a/rest/doc_api_test.go b/rest/doc_api_test.go index 5ab381b547..ecf3264393 100644 --- a/rest/doc_api_test.go +++ b/rest/doc_api_test.go @@ -12,7 +12,10 @@ package rest import ( "fmt" + "io" "log" + "mime" + "mime/multipart" "net/http" "strings" "testing" @@ -173,3 +176,129 @@ func TestGuestReadOnly(t *testing.T) { RequireStatus(t, response, http.StatusForbidden) } + +func TestGetDocWithCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + docID := "doc1" + docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + testCases := []struct { + name string + url string + output string + headers map[string]string + multipart bool + }{ + { + name: "get doc", + url: "/{{.keyspace}}/doc1", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, docID, docVersion.RevTreeID), + }, + { + name: "get doc with rev", + url: fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s", docVersion.RevTreeID), + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, docID, docVersion.RevTreeID), + }, + { + name: "get doc with cv", + url: "/{{.keyspace}}/doc1?show_cv=true", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}`, docID, docVersion.RevTreeID, docVersion.CV), + }, + { + name: "get doc with open_revs=all and cv no multipart", + url: "/{{.keyspace}}/doc1?open_revs=all&show_cv=true", + output: fmt.Sprintf(`[{"ok": {"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}}]`, docID, docVersion.RevTreeID, docVersion.CV), + headers: map[string]string{ + "Accept": "application/json", + }, + }, + + { + name: "get doc with open_revs=all and cv", + url: "/{{.keyspace}}/doc1?open_revs=all&show_cv=true", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}`, docID, docVersion.RevTreeID, docVersion.CV), + multipart: true, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequestWithHeaders("GET", testCase.url, "", testCase.headers) + RequireStatus(t, response, http.StatusOK) + output := response.BodyString() + if testCase.multipart { + multipartOutput := readMultiPartBody(t, response) + require.Len(t, multipartOutput, 1) + output = multipartOutput[0] + } + assert.JSONEq(t, testCase.output, output) + }) + } + +} + +func TestBulkGetWithCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + doc1ID := "doc1" + doc2ID := "doc2" + doc1Version := rt.PutDocDirectly(doc1ID, db.Body{"foo": "bar"}) + doc2Version := rt.PutDocDirectly(doc2ID, db.Body{"foo": "baz"}) + testCases := []struct { + name string + url string + input string + output []string + }{ + { + name: "get doc multipart", + url: "/{{.keyspace}}/_bulk_get", + input: fmt.Sprintf(`{"docs":[{"id":"%s"},{"id":"%s"}]}`, doc1ID, doc2ID), + output: []string{ + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, doc1ID, doc1Version.RevTreeID), + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"baz"}`, doc2ID, doc2Version.RevTreeID), + }, + }, + { + name: "get doc multipart", + url: "/{{.keyspace}}/_bulk_get?show_cv=true", + input: fmt.Sprintf(`{"docs":[{"id":"%s"},{"id":"%s"}]}`, doc1ID, doc2ID), + output: []string{ + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar", "_cv": "%s"}`, doc1ID, doc1Version.RevTreeID, doc1Version.CV), + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"baz", "_cv": "%s"}`, doc2ID, doc2Version.RevTreeID, doc2Version.CV), + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequest(http.MethodPost, testCase.url, testCase.input) + RequireStatus(t, response, http.StatusOK) + bodies := readMultiPartBody(t, response) + require.Len(t, bodies, len(testCase.output)) + for i, body := range bodies { + assert.JSONEq(t, testCase.output[i], body) + } + }) + } + +} + +// readMultiPartBody reads a multipart response body and returns the parts as strings +func readMultiPartBody(t *testing.T, response *TestResponse) []string { + _, params, err := mime.ParseMediaType(response.Header().Get("Content-Type")) + require.NoError(t, err) + mr := multipart.NewReader(response.Body, params["boundary"]) + var output []string + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + bodyBytes, err := io.ReadAll(p) + require.NoError(t, err) + output = append(output, string(bodyBytes)) + } + return output +}