From d6a20e016a6c0aec5b035a93ec907f730403fd27 Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Fri, 16 Feb 2024 10:57:11 -0800 Subject: [PATCH] CBG-3788 Support HLV operations in BlipTesterClient (#6689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CBG-3788 Support HLV operations in BlipTesterClient Switches the BlipTesterCollectionClient to maintain client HLV and (linear) revtree per document. Switches the docs struct to a map of a new BlipTesterDoc struct, instead of a map of revs per document. BlipTesterDoc still maintains a history of all rev messages received (revMessageHistory) to support test evaluation of received messages, but also defines a linear revTreeId history or an HLV (depending on protocol enabled for the test). Includes a refactor of revID to revTreeID in RevAndVersion, as a step toward standardizing ‘revID’ as the generic property used during replication (which can be currentRev or cv), and revTreeID as a traditional revtree revision ID. * Fixes based on PR review --- db/blip_sync_context.go | 2 +- db/crud.go | 2 +- db/hybrid_logical_vector.go | 24 +- db/hybrid_logical_vector_test.go | 2 +- db/revision_cache_bypass.go | 6 +- db/revision_cache_lru.go | 8 +- rest/access_test.go | 20 +- rest/adminapitest/admin_api_test.go | 10 +- rest/api_test.go | 4 +- rest/api_test_helpers.go | 6 +- rest/attachment_test.go | 56 +-- rest/blip_api_attachment_test.go | 9 +- rest/blip_api_crud_test.go | 17 +- rest/blip_api_delta_sync_test.go | 6 +- rest/blip_client_test.go | 399 +++++++++++------- rest/bulk_api.go | 2 +- rest/changes_test.go | 4 +- rest/changestest/changes_api_test.go | 10 +- rest/importtest/import_test.go | 4 +- rest/replicatortest/replicator_test.go | 14 +- rest/replicatortest/replicator_test_helper.go | 2 +- rest/revocation_test.go | 4 +- rest/utilities_testing.go | 20 +- rest/utilities_testing_resttester.go | 32 +- 24 files changed, 394 insertions(+), 269 deletions(-) diff --git a/db/blip_sync_context.go b/db/blip_sync_context.go index 828f80fb68..b7e05e84f9 100644 --- a/db/blip_sync_context.go +++ b/db/blip_sync_context.go @@ -623,7 +623,7 @@ func (bsc *BlipSyncContext) sendRevision(sender *blip.Sender, docID, rev string, docRev, err = handleChangesResponseCollection.GetRev(bsc.loggingCtx, docID, rev, true, nil) } else { // extract CV string rev representation - version, vrsErr := CreateVersionFromString(rev) + version, vrsErr := ParseVersion(rev) if vrsErr != nil { return vrsErr } diff --git a/db/crud.go b/db/crud.go index 716bfafa12..8ed1d02dd3 100644 --- a/db/crud.go +++ b/db/crud.go @@ -2233,7 +2233,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do Expiry: doc.Expiry, Deleted: doc.History[newRevID].Deleted, _shallowCopyBody: storedDoc.Body(ctx), - hlvHistory: doc.HLV.toHistoryForHLV(), + hlvHistory: doc.HLV.ToHistoryForHLV(), CV: &Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}, } diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 90d0c572c2..2648c1a560 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -70,7 +70,7 @@ func CreateVersion(source, version string) Version { } } -func CreateVersionFromString(versionString string) (version Version, err error) { +func ParseVersion(versionString string) (version Version, err error) { timestampString, sourceBase64, found := strings.Cut(versionString, "@") if !found { return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString) @@ -80,6 +80,16 @@ func CreateVersionFromString(versionString string) (version Version, err error) return version, nil } +func ParseDecodedVersion(versionString string) (version DecodedVersion, err error) { + timestampString, sourceBase64, found := strings.Cut(versionString, "@") + if !found { + return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString) + } + version.SourceID = sourceBase64 + version.Value = base.HexCasToUint64(timestampString) + return version, nil +} + // String returns a Couchbase Lite-compatible string representation of the version. func (v DecodedVersion) String() string { timestamp := string(base.Uint64CASToLittleEndianHex(v.Value)) @@ -178,7 +188,7 @@ func (hlv *HybridLogicalVector) AddVersion(newVersion Version) error { if currPVVersionCAS < hlvVersionCAS { hlv.PreviousVersions[hlv.SourceID] = hlv.Version } else { - return fmt.Errorf("local hlv has current source in previous versiosn with version greater than current version. Current CAS: %s, PV CAS %s", hlv.Version, currPVVersion) + return fmt.Errorf("local hlv has current source in previous version with version greater than current version. Current CAS: %s, PV CAS %s", hlv.Version, currPVVersion) } } else { // source doesn't exist in PV so add @@ -375,8 +385,16 @@ func (hlv *HybridLogicalVector) setPreviousVersion(source string, version string hlv.PreviousVersions[source] = version } +func (hlv *HybridLogicalVector) IsVersionKnown(otherVersion Version) bool { + value, found := hlv.GetVersion(otherVersion.SourceID) + if !found { + return false + } + return value >= base.HexCasToUint64(otherVersion.Value) +} + // toHistoryForHLV formats blip History property for V4 replication and above -func (hlv *HybridLogicalVector) toHistoryForHLV() string { +func (hlv *HybridLogicalVector) ToHistoryForHLV() string { // take pv and mv from hlv if defined and add to history var s strings.Builder // Merge versions must be defined first if they exist diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index a00ca31254..3e4a94b885 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -332,7 +332,7 @@ func TestHLVMapToCBLString(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { hlv := createHLVForTest(t, test.inputHLV) - historyStr := hlv.toHistoryForHLV() + historyStr := hlv.ToHistoryForHLV() if test.both { initial := strings.Split(historyStr, ";") diff --git a/db/revision_cache_bypass.go b/db/revision_cache_bypass.go index e297467cc0..2aa2daa2a4 100644 --- a/db/revision_cache_bypass.go +++ b/db/revision_cache_bypass.go @@ -52,7 +52,7 @@ func (rc *BypassRevisionCache) GetWithRev(ctx context.Context, docID, revID stri } if hlv != nil { docRev.CV = hlv.ExtractCurrentVersionFromHLV() - docRev.hlvHistory = hlv.toHistoryForHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() } rc.bypassStat.Add(1) @@ -80,7 +80,7 @@ func (rc *BypassRevisionCache) GetWithCV(ctx context.Context, docID string, cv * } if hlv != nil { docRev.CV = hlv.ExtractCurrentVersionFromHLV() - docRev.hlvHistory = hlv.toHistoryForHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() } rc.bypassStat.Add(1) @@ -111,7 +111,7 @@ func (rc *BypassRevisionCache) GetActive(ctx context.Context, docID string, incl } if hlv != nil { docRev.CV = hlv.ExtractCurrentVersionFromHLV() - docRev.hlvHistory = hlv.toHistoryForHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() } rc.bypassStat.Add(1) diff --git a/db/revision_cache_lru.go b/db/revision_cache_lru.go index c31232d889..59a94d821d 100644 --- a/db/revision_cache_lru.go +++ b/db/revision_cache_lru.go @@ -544,7 +544,7 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache // based off the current value load we need to populate the revid key with what has been fetched from the bucket (for use of populating the opposite lookup map) value.revID = revid if hlv != nil { - value.hlvHistory = hlv.toHistoryForHLV() + value.hlvHistory = hlv.ToHistoryForHLV() } } else { revKey := IDAndRev{DocID: value.id, RevID: value.revID} @@ -552,7 +552,7 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache // based off the revision load we need to populate the hlv key with what has been fetched from the bucket (for use of populating the opposite lookup map) if hlv != nil { value.cv = *hlv.ExtractCurrentVersionFromHLV() - value.hlvHistory = hlv.toHistoryForHLV() + value.hlvHistory = hlv.ToHistoryForHLV() } } } @@ -655,12 +655,12 @@ func (value *revCacheValue) loadForDoc(ctx context.Context, backingStore Revisio if value.revID == "" { value.bodyBytes, value.body, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, revid, hlv, value.err = revCacheLoaderForDocumentCV(ctx, backingStore, doc, value.cv) value.revID = revid - value.hlvHistory = hlv.toHistoryForHLV() + value.hlvHistory = hlv.ToHistoryForHLV() } else { value.bodyBytes, value.body, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, hlv, value.err = revCacheLoaderForDocument(ctx, backingStore, doc, value.revID) if hlv != nil { value.cv = *hlv.ExtractCurrentVersionFromHLV() - value.hlvHistory = hlv.toHistoryForHLV() + value.hlvHistory = hlv.ToHistoryForHLV() } } } diff --git a/rest/access_test.go b/rest/access_test.go index 8abbee7a55..0fdd9c8cd1 100644 --- a/rest/access_test.go +++ b/rest/access_test.go @@ -414,11 +414,11 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // User has no permissions to access rev - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusOK) // Guest has no permissions to access rev - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusOK) // Attachments should be forbidden as well @@ -426,7 +426,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // Attachment revs should be forbidden as well - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusNotFound) // Attachments should be forbidden for guests as well @@ -434,7 +434,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // Attachment revs should be forbidden for guests as well - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusNotFound) // Document does not exist should cause 403 @@ -451,7 +451,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // PUT with rev - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "NoPerms", "password") assertRespStatus(resp, http.StatusForbidden) // PUT with incorrect rev @@ -463,7 +463,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // PUT with rev as Guest - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "", "") assertRespStatus(resp, http.StatusForbidden) // PUT with incorrect rev as Guest @@ -495,7 +495,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assert.NotContains(t, user.GetChannels(s, c).ToArray(), "chan2") // Successful PUT which will grant access grants - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{"channels": "chan"}`, nil, "Perms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{"channels": "chan"}`, nil, "Perms", "password") AssertStatus(t, resp, http.StatusCreated) // Make sure channel access grant was successful @@ -512,7 +512,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // Attempt to delete document rev with no permissions - resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusConflict) // Attempt to delete document with wrong rev @@ -528,7 +528,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // Attempt to delete document rev with no write perms as guest - resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusConflict) // Attempt to delete document with wrong rev as guest @@ -1184,7 +1184,7 @@ func TestRoleChannelGrantInheritance(t *testing.T) { RequireStatus(t, response, 200) // Revoke access to chan2 (dynamic) - response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1") + response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevTreeID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1") RequireStatus(t, response, 201) // Verify user cannot access doc in revoked channel, but can successfully access remaining documents diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index 99160bf19e..89d13773bc 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -2144,7 +2144,7 @@ func TestRawTombstone(t *testing.T) { resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``) assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`) - assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevID+`"`) + assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevTreeID+`"`) assert.Contains(t, string(resp.BodyBytes()), `"foo":"bar"`) assert.NotContains(t, string(resp.BodyBytes()), `"_deleted":true`) @@ -2154,7 +2154,7 @@ func TestRawTombstone(t *testing.T) { resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``) assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`) - assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevID+`"`) + assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevTreeID+`"`) assert.NotContains(t, string(resp.BodyBytes()), `"foo":"bar"`) assert.Contains(t, string(resp.BodyBytes()), `"_deleted":true`) } @@ -3881,9 +3881,9 @@ func TestPutIDRevMatchBody(t *testing.T) { docRev := test.rev docBody := test.docBody if test.docID == "" { - docID = "doc" // Used for the rev tests to branch off of - docBody = strings.ReplaceAll(docBody, "[REV]", version.RevID) // FIX for HLV? - docRev = strings.ReplaceAll(docRev, "[REV]", version.RevID) + docID = "doc" // Used for the rev tests to branch off of + docBody = strings.ReplaceAll(docBody, "[REV]", version.RevTreeID) // FIX for HLV? + docRev = strings.ReplaceAll(docRev, "[REV]", version.RevTreeID) } resp := rt.SendAdminRequest("PUT", fmt.Sprintf("/{{.keyspace}}/%s?rev=%s", docID, docRev), docBody) diff --git a/rest/api_test.go b/rest/api_test.go index ebd4b68ccb..588d7fc4d7 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -211,9 +211,9 @@ func TestDocLifecycle(t *testing.T) { defer rt.Close() version := rt.CreateTestDoc("doc") - assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevID) + assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevTreeID) - response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevID, "") + response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevTreeID, "") RequireStatus(t, response, 200) } diff --git a/rest/api_test_helpers.go b/rest/api_test_helpers.go index 9bca5857c7..0748914c8e 100644 --- a/rest/api_test_helpers.go +++ b/rest/api_test_helpers.go @@ -50,13 +50,13 @@ func (rt *RestTester) PutNewEditsFalse(docID string, newVersion DocVersion, pare require.NoError(rt.TB, marshalErr) requestBody := body.ShallowCopy() - newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB), newVersion.RevID) + newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB), newVersion.RevTreeID) revisions := make(map[string]interface{}) revisions["start"] = newRevGeneration ids := []string{newRevDigest} - if parentVersion.RevID != "" { - _, parentDigest := db.ParseRevID(base.TestCtx(rt.TB), parentVersion.RevID) + if parentVersion.RevTreeID != "" { + _, parentDigest := db.ParseRevID(base.TestCtx(rt.TB), parentVersion.RevTreeID) ids = append(ids, parentDigest) } revisions["ids"] = ids diff --git a/rest/attachment_test.go b/rest/attachment_test.go index b528c413cb..b03b4d41f7 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -43,25 +43,25 @@ func TestDocEtag(t *testing.T) { version := DocVersionFromPutResponse(t, response) // Validate Etag returned on doc creation - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) response = rt.SendRequest("GET", "/{{.keyspace}}/doc", "") RequireStatus(t, response, 200) // Validate Etag returned when retrieving doc - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) // Validate Etag returned when updating doc - response = rt.SendRequest("PUT", "/{{.keyspace}}/doc?rev="+version.RevID, `{"prop":false}`) + response = rt.SendRequest("PUT", "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{"prop":false}`) version = DocVersionFromPutResponse(t, response) - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) // Test Attachments attachmentBody := "this is the body of attachment" attachmentContentType := "content/type" // attach to existing document with correct rev (should succeed), manual request to change etag - resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", "doc", "attach1", version.RevID) + resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", "doc", "attach1", version.RevTreeID) response = rt.SendAdminRequestWithHeaders(http.MethodPut, resource, attachmentBody, attachmentHeaders()) RequireStatus(t, response, http.StatusCreated) var body db.Body @@ -71,7 +71,7 @@ func TestDocEtag(t *testing.T) { RequireDocVersionNotEqual(t, version, afterAttachmentVersion) // validate Etag returned from adding an attachment - assert.Equal(t, strconv.Quote(afterAttachmentVersion.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(afterAttachmentVersion.RevTreeID), response.Header().Get("Etag")) // retrieve attachment response = rt.SendRequest("GET", "/{{.keyspace}}/doc/attach1", "") @@ -115,7 +115,7 @@ func TestDocAttachment(t *testing.T) { assert.Equal(t, attachmentContentType, response.Header().Get("Content-Type")) // attempt to delete an attachment that is not on the document - response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach2?rev="+version.RevID, "") + response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach2?rev="+version.RevTreeID, "") RequireStatus(t, response, 404) // attempt to delete attachment from non existing doc @@ -127,7 +127,7 @@ func TestDocAttachment(t *testing.T) { RequireStatus(t, response, 409) // delete the attachment calling the delete attachment endpoint - response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach1?rev="+version.RevID, "") + response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach1?rev="+version.RevTreeID, "") RequireStatus(t, response, 200) // attempt to access deleted attachment (should return error) @@ -221,7 +221,7 @@ func TestDocAttachmentOnRemovedRev(t *testing.T) { } // attach to existing document with correct rev (should fail) - response := rt.SendUserRequestWithHeaders("PUT", "/{{.keyspace}}/doc/attach1?rev="+version.RevID, attachmentBody, reqHeaders, "user1", "letmein") + response := rt.SendUserRequestWithHeaders("PUT", "/{{.keyspace}}/doc/attach1?rev="+version.RevTreeID, attachmentBody, reqHeaders, "user1", "letmein") RequireStatus(t, response, 404) } @@ -429,7 +429,7 @@ func TestAttachmentsNoCrossTalk(t *testing.T) { "Accept": "application/json", } - response := rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevID, doc1Version.RevID), "", reqHeaders) + response := rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevTreeID, doc1Version.RevTreeID), "", reqHeaders) assert.Equal(t, 200, response.Code) // validate attachment has data property body := db.Body{} @@ -440,7 +440,7 @@ func TestAttachmentsNoCrossTalk(t *testing.T) { data := attach1["data"] assert.True(t, data != nil) - response = rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevID, afterAttachmentVersion.RevID), "", reqHeaders) + response = rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevTreeID, afterAttachmentVersion.RevTreeID), "", reqHeaders) assert.Equal(t, 200, response.Code) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) log.Printf("response body revid1 = %s", body) @@ -593,7 +593,7 @@ func TestBulkGetBadAttachmentReproIssue2528(t *testing.T) { version, _ := rt.GetDoc(doc1ID) // Do a bulk_get to get the doc -- this was causing a panic prior to the fix for #2528 - bulkGetDocs := fmt.Sprintf(`{"docs": [{"id": "%v", "rev": "%v"}, {"id": "%v", "rev": "%v"}]}`, doc1ID, version.RevID, doc2ID, doc2Version.RevID) + bulkGetDocs := fmt.Sprintf(`{"docs": [{"id": "%v", "rev": "%v"}, {"id": "%v", "rev": "%v"}]}`, doc1ID, version.RevTreeID, doc2ID, doc2Version.RevTreeID) bulkGetResponse := rt.SendAdminRequest("POST", "/{{.keyspace}}/_bulk_get?revs=true&attachments=true&revs_limit=2", bulkGetDocs) if bulkGetResponse.Code != 200 { panic(fmt.Sprintf("Got unexpected response: %v", bulkGetResponse)) @@ -698,15 +698,15 @@ func TestConflictWithInvalidAttachment(t *testing.T) { // Set attachment attachmentBody := "aGVsbG8gd29ybGQ=" // hello.txt - response := rt.SendAdminRequestWithHeaders("PUT", "/{{.keyspace}}/doc1/attach1?rev="+version.RevID, attachmentBody, reqHeaders) + response := rt.SendAdminRequestWithHeaders("PUT", "/{{.keyspace}}/doc1/attach1?rev="+version.RevTreeID, attachmentBody, reqHeaders) RequireStatus(t, response, http.StatusCreated) - docrevId2 := DocVersionFromPutResponse(t, response).RevID + docrevId2 := DocVersionFromPutResponse(t, response).RevTreeID // Update Doc rev3Input := `{"_attachments":{"attach1":{"content-type": "content/type", "digest":"sha1-b7fDq/pHG8Nf5F3fe0K2nu0xcw0=", "length": 16, "revpos": 2, "stub": true}}, "_id": "doc1", "_rev": "` + docrevId2 + `", "prop":true}` response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1", rev3Input) RequireStatus(t, response, http.StatusCreated) - docrevId3 := DocVersionFromPutResponse(t, response).RevID + docrevId3 := DocVersionFromPutResponse(t, response).RevTreeID // Get Existing Doc & Update rev rev4Input := `{"_attachments":{"attach1":{"content-type": "content/type", "digest":"sha1-b7fDq/pHG8Nf5F3fe0K2nu0xcw0=", "length": 16, "revpos": 2, "stub": true}}, "_id": "doc1", "_rev": "` + docrevId3 + `", "prop":true}` @@ -796,7 +796,7 @@ func TestConflictingBranchAttachments(t *testing.T) { response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev2a) RequireStatus(t, response, http.StatusCreated) docVersion2a := DocVersionFromPutResponse(t, response) - assert.Equal(t, "2-twoa", docVersion2a.RevID) + assert.Equal(t, "2-twoa", docVersion2a.RevTreeID) // Put attachment on doc1 rev 2 rev3Attachment := `aGVsbG8gd29ybGQ=` // hello.txt @@ -815,8 +815,8 @@ func TestConflictingBranchAttachments(t *testing.T) { docVersion4a := rt.UpdateDoc("doc1", docVersion3a, rev4aBody) // Ensure the two attachments are different - response1 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]&rev="+docVersion4.RevID, "") - response2 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?rev="+docVersion4a.RevID, "") + response1 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]&rev="+docVersion4.RevTreeID, "") + response2 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?rev="+docVersion4a.RevTreeID, "") var body1 db.Body var body2 db.Body @@ -865,14 +865,14 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { `}` _ = rt.UpdateDoc("doc1", docVersion5, rev6Body) - response := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") log.Printf("Rev6 GET: %s", response.Body.Bytes()) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) _, attachmentsPresent := body["_attachments"] assert.True(t, attachmentsPresent) // Create conflicting rev 6 that doesn't have attachments - reqBodyRev6a := `{"_rev": "6-a", "_revisions": {"ids": ["a", "` + docVersion5.RevID + `"], "start": 6}}` + reqBodyRev6a := `{"_rev": "6-a", "_revisions": {"ids": ["a", "` + docVersion5.RevTreeID + `"], "start": 6}}` response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev6a) RequireStatus(t, response, http.StatusCreated) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) @@ -880,7 +880,7 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { assert.Equal(t, "6-a", docRevId2a) var rev6Response db.Body - response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &rev6Response)) _, attachmentsPresent = rev6Response["_attachments"] assert.False(t, attachmentsPresent) @@ -891,7 +891,7 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { // Retrieve current winning rev with attachments var rev7Response db.Body - response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") log.Printf("Rev6 GET: %s", response.Body.Bytes()) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &rev7Response)) _, attachmentsPresent = rev7Response["_attachments"] @@ -2117,7 +2117,7 @@ func TestAttachmentRemovalWithConflicts(t *testing.T) { var doc1 docResp // Get losing rev and ensure attachment is still there and has not been deleted - resp := rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+losingVersion3.RevID, "", map[string]string{"Accept": "application/json"}) + resp := rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+losingVersion3.RevTreeID, "", map[string]string{"Accept": "application/json"}) RequireStatus(t, resp, http.StatusOK) err = base.JSONUnmarshal(resp.BodyBytes(), &doc1) @@ -2134,7 +2134,7 @@ func TestAttachmentRemovalWithConflicts(t *testing.T) { var doc2 docResp // Get winning rev and ensure attachment is indeed removed from this rev - resp = rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+finalVersion4.RevID, "", map[string]string{"Accept": "application/json"}) + resp = rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+finalVersion4.RevTreeID, "", map[string]string{"Accept": "application/json"}) RequireStatus(t, resp, http.StatusOK) err = base.JSONUnmarshal(resp.BodyBytes(), &doc2) @@ -2401,7 +2401,7 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { // Push a revision with a bunch of history simulating doc updated on mobile device // Note this references revpos 1 and therefore SGW has it - Shouldn't need proveAttachment proveAttachmentBefore := btc.pushReplication.replicationStats.ProveAttachment.Value() - revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.RevID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) + revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.RevTreeID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) assert.NoError(t, err) proveAttachmentAfter := btc.pushReplication.replicationStats.ProveAttachment.Value() assert.Equal(t, proveAttachmentBefore, proveAttachmentAfter) @@ -2446,7 +2446,7 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { btcRunner.AttachmentsLock(btc.id).Unlock() // Put doc with an erroneous revpos 1 but with a different digest, referring to the above attachment - _, err = btcRunner.PushRevWithHistory(btc.id, docID, version.RevID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) + _, err = btcRunner.PushRevWithHistory(btc.id, docID, version.RevTreeID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) require.NoError(t, err) // Ensure message and attachment is pushed up @@ -2749,7 +2749,7 @@ func (rt *RestTester) storeAttachment(docID string, version DocVersion, attName, // storeAttachmentWithHeaders adds an attachment to a document version and returns the new version using rev= syntax. func (rt *RestTester) storeAttachmentWithHeaders(docID string, version DocVersion, attName, attBody string, reqHeaders map[string]string) DocVersion { - resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", docID, attName, version.RevID) + resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", docID, attName, version.RevTreeID) response := rt.SendAdminRequestWithHeaders(http.MethodPut, resource, attBody, reqHeaders) RequireStatus(rt.TB, response, http.StatusCreated) var body db.Body @@ -2761,7 +2761,7 @@ func (rt *RestTester) storeAttachmentWithHeaders(docID string, version DocVersio // storeAttachmentWithIfMatch adds an attachment to a document version and returns the new version, using If-Match. func (rt *RestTester) storeAttachmentWithIfMatch(docID string, version DocVersion, attName, attBody string) DocVersion { reqHeaders := attachmentHeaders() - reqHeaders["If-Match"] = `"` + version.RevID + `"` + reqHeaders["If-Match"] = `"` + version.RevTreeID + `"` resource := fmt.Sprintf("/{{.keyspace}}/%s/%s", docID, attName) response := rt.SendRequestWithHeaders(http.MethodPut, resource, attBody, reqHeaders) RequireStatus(rt.TB, response, http.StatusCreated) diff --git a/rest/blip_api_attachment_test.go b/rest/blip_api_attachment_test.go index a895fd9b17..d35941800b 100644 --- a/rest/blip_api_attachment_test.go +++ b/rest/blip_api_attachment_test.go @@ -291,6 +291,7 @@ func TestBlipPushPullNewAttachmentCommonAncestor(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { @@ -366,6 +367,7 @@ func TestBlipPushPullNewAttachmentNoCommonAncestor(t *testing.T) { const docID = "doc1" btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -530,6 +532,7 @@ func TestBlipAttachNameChange(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -579,6 +582,7 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -601,7 +605,7 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { docVersion, _ := client1.rt.GetDoc(docID) // Store the document and attachment on the test client - err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.RevID, rawDoc) + err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.RevTreeID, rawDoc) require.NoError(t, err) btcRunner.AttachmentsLock(client1.id).Lock() @@ -636,6 +640,7 @@ func TestBlipLegacyAttachDocUpdate(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -658,7 +663,7 @@ func TestBlipLegacyAttachDocUpdate(t *testing.T) { version, _ := client1.rt.GetDoc(docID) // Store the document and attachment on the test client - err := btcRunner.StoreRevOnClient(client1.id, docID, version.RevID, rawDoc) + err := btcRunner.StoreRevOnClient(client1.id, docID, version.RevTreeID, rawDoc) require.NoError(t, err) btcRunner.AttachmentsLock(client1.id).Lock() btcRunner.Attachments(client1.id)[digest] = attBody diff --git a/rest/blip_api_crud_test.go b/rest/blip_api_crud_test.go index c8fbefc1bb..b443f34478 100644 --- a/rest/blip_api_crud_test.go +++ b/rest/blip_api_crud_test.go @@ -651,21 +651,21 @@ func TestProposedChangesIncludeConflictingRev(t *testing.T) { // Write existing docs to server directly (not via blip) rt := bt.restTester resp := rt.PutDoc("conflictingInsert", `{"version":1}`) - conflictingInsertRev := resp.RevID + conflictingInsertRev := resp.RevTreeID resp = rt.PutDoc("matchingInsert", `{"version":1}`) - matchingInsertRev := resp.RevID + matchingInsertRev := resp.RevTreeID resp = rt.PutDoc("conflictingUpdate", `{"version":1}`) - conflictingUpdateRev1 := resp.RevID - conflictingUpdateRev2 := rt.UpdateDocRev("conflictingUpdate", resp.RevID, `{"version":2}`) + conflictingUpdateRev1 := resp.RevTreeID + conflictingUpdateRev2 := rt.UpdateDocRev("conflictingUpdate", resp.RevTreeID, `{"version":2}`) resp = rt.PutDoc("matchingUpdate", `{"version":1}`) - matchingUpdateRev1 := resp.RevID - matchingUpdateRev2 := rt.UpdateDocRev("matchingUpdate", resp.RevID, `{"version":2}`) + matchingUpdateRev1 := resp.RevTreeID + matchingUpdateRev2 := rt.UpdateDocRev("matchingUpdate", resp.RevTreeID, `{"version":2}`) resp = rt.PutDoc("newUpdate", `{"version":1}`) - newUpdateRev1 := resp.RevID + newUpdateRev1 := resp.RevTreeID type proposeChangesCase struct { key string @@ -1953,7 +1953,7 @@ func TestPullReplicationUpdateOnOtherHLVAwarePeer(t *testing.T) { // create doc version of the above doc write version1 := DocVersion{ - RevID: bucketDoc.CurrentRev, + RevTreeID: bucketDoc.CurrentRev, CV: db.Version{ SourceID: hlvHelper.Source, Value: string(base.Uint64CASToLittleEndianHex(cas)), @@ -2479,6 +2479,7 @@ func TestBlipInternalPropertiesHandling(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { // Setup diff --git a/rest/blip_api_delta_sync_test.go b/rest/blip_api_delta_sync_test.go index 9908f886d8..a1bfe85d7a 100644 --- a/rest/blip_api_delta_sync_test.go +++ b/rest/blip_api_delta_sync_test.go @@ -42,6 +42,8 @@ func TestBlipDeltaSyncPushAttachment(t *testing.T) { const docID = "pushAttachmentDoc" btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires push replication (CBG-3255) + btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) defer rt.Close() @@ -373,7 +375,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { defer client.Close() // reject deltas built ontop of rev 1 - client.rejectDeltasForSrcRev = docVersion1.RevID + client.rejectDeltasForSrcRev = docVersion1.RevTreeID client.ClientDeltas = true err := btcRunner.StartPull(client.id) @@ -390,7 +392,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { msg := client.pullReplication.WaitForMessage(5) // Check the request was initially sent with the correct deltaSrc property - assert.Equal(t, docVersion1.RevID, msg.Properties[db.RevMessageDeltaSrc]) + assert.Equal(t, docVersion1.RevTreeID, msg.Properties[db.RevMessageDeltaSrc]) // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) diff --git a/rest/blip_client_test.go b/rest/blip_client_test.go index 5676f61736..0c0d88816d 100644 --- a/rest/blip_client_test.go +++ b/rest/blip_client_test.go @@ -65,18 +65,13 @@ type BlipTesterClient struct { } type BlipTesterCollectionClient struct { - parent *BlipTesterClient - - collection string - collectionIdx int - - docs map[string]map[string]*BodyMessagePair // Client's local store of documents - Map of docID - // to rev ID to bytes - attachments map[string][]byte // Client's local store of attachments - Map of digest to bytes - lastReplicatedRev map[string]string // Latest known rev pulled or pushed - docsLock sync.RWMutex // lock for docs map - attachmentsLock sync.RWMutex // lock for attachments map - lastReplicatedRevLock sync.RWMutex // lock for lastReplicatedRev map + parent *BlipTesterClient + collection string + collectionIdx int + docs map[string]*BlipTesterDoc // Client's local store of documents, indexed by DocID + attachments map[string][]byte // Client's local store of attachments - Map of digest to bytes + docsLock sync.RWMutex // lock for docs map + attachmentsLock sync.RWMutex // lock for attachments map } // BlipTestClientRunner is for running the blip tester client and its associated methods in test framework @@ -87,9 +82,94 @@ type BlipTestClientRunner struct { SkipSubtest map[string]bool // map of sub tests on the blip tester runner to skip } -type BodyMessagePair struct { - body []byte - message *blip.Message +type BlipTesterDoc struct { + revMode blipTesterRevMode + body []byte + revMessageHistory map[string]*blip.Message // History of rev messages received for this document, indexed by revID + revHistory []string // ordered history of revTreeIDs (newest first), populated when mode = revtree + HLV db.HybridLogicalVector // HLV, populated when mode = HLV +} + +const ( + revModeRevTree blipTesterRevMode = iota + revModeHLV +) + +type blipTesterRevMode uint32 + +func (doc *BlipTesterDoc) isRevKnown(revID string) bool { + if doc.revMode == revModeHLV { + version := VersionFromRevID(revID) + return doc.HLV.IsVersionKnown(version) + } else { + for _, revTreeID := range doc.revHistory { + if revTreeID == revID { + return true + } + } + } + return false +} + +func (doc *BlipTesterDoc) makeRevHistoryForChangesResponse() []string { + if doc.revMode == revModeHLV { + // For HLV, a changes response only needs to send cv, since rev message will always send full HLV + return []string{doc.HLV.GetCurrentVersionString()} + } else { + var revList []string + if len(doc.revHistory) < 20 { + revList = doc.revHistory + } else { + revList = doc.revHistory[0:19] + } + return revList + } +} + +func (doc *BlipTesterDoc) getCurrentRevID() string { + if doc.revMode == revModeHLV { + return doc.HLV.GetCurrentVersionString() + } else { + if len(doc.revHistory) == 0 { + return "" + } + return doc.revHistory[0] + } +} + +func (doc *BlipTesterDoc) addRevision(revID string, body []byte, message *blip.Message) { + doc.revMessageHistory[revID] = message + doc.body = body + if doc.revMode == revModeHLV { + _ = doc.HLV.AddVersion(VersionFromRevID(revID)) + } else { + // prepend revID to revHistory + doc.revHistory = append([]string{revID}, doc.revHistory...) + } +} + +func (btcr *BlipTesterCollectionClient) NewBlipTesterDoc(revID string, body []byte, message *blip.Message) *BlipTesterDoc { + doc := &BlipTesterDoc{ + body: body, + revMessageHistory: map[string]*blip.Message{revID: message}, + } + if btcr.UseHLV() { + doc.revMode = revModeHLV + doc.HLV = db.NewHybridLogicalVector() + _ = doc.HLV.AddVersion(VersionFromRevID(revID)) + } else { + doc.revMode = revModeRevTree + doc.revHistory = []string{revID} + } + return doc +} + +func VersionFromRevID(revID string) db.Version { + version, err := db.ParseVersion(revID) + if err != nil { + panic(err) + } + return version } // BlipTesterReplicator is a BlipTester which stores a map of messages keyed by Serial Number @@ -150,7 +230,6 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { btr.bt.blipContext.HandlerForProfile[db.MessageChanges] = func(msg *blip.Message) { btr.storeMessage(msg) - btcr := btc.getCollectionClientFromMessage(msg) // Exit early when there's nothing to do @@ -186,33 +265,16 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { // Build up a list of revisions known to the client for each change // The first element of each revision list must be the parent revision of the change - if revs, haveDoc := btcr.docs[docID]; haveDoc { - revList := make([]string, 0, len(revs)) - - // Insert the highest ancestor rev generation at the start of the revList - latest, ok := btcr.getLastReplicatedRev(docID) - if ok { - revList = append(revList, latest) + if doc, haveDoc := btcr.docs[docID]; haveDoc { + if deletedInt&2 == 2 { + continue } - for knownRevID := range revs { - if deletedInt&2 == 2 { - continue - } - - if revID == knownRevID { - knownRevs[i] = nil // Send back null to signal we don't need this change - continue outer - } else if latest == knownRevID { - // We inserted this rev as the first element above, so skip it here - continue - } - - // TODO: Limit known revs to 20 to copy CBL behaviour - revList = append(revList, knownRevID) + if doc.isRevKnown(revID) { + knownRevs[i] = nil + continue outer } - - knownRevs[i] = revList + knownRevs[i] = doc.makeRevHistoryForChangesResponse() } else { knownRevs[i] = []interface{}{} // sending empty array means we've not seen the doc before, but still want it } @@ -252,14 +314,12 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { if msg.Properties[db.RevMessageDeleted] == "1" { btcr.docsLock.Lock() defer btcr.docsLock.Unlock() - if _, ok := btcr.docs[docID]; ok { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID][revID] = bodyMessagePair + + if doc, ok := btcr.docs[docID]; ok { + doc.addRevision(revID, body, msg) } else { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} + btcr.docs[docID] = btcr.NewBlipTesterDoc(revID, body, msg) } - btcr.updateLastReplicatedRev(docID, revID) if !msg.NoReply() { response := msg.Response() @@ -290,7 +350,12 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { var old db.Body btcr.docsLock.RLock() - oldBytes := btcr.docs[docID][deltaSrc].body + // deltaSrc must be the current rev + doc := btcr.docs[docID] + if doc.getCurrentRevID() != deltaSrc { + panic("current rev doesn't match deltaSrc") + } + oldBytes := doc.body btcr.docsLock.RUnlock() err = old.Unmarshal(oldBytes) require.NoError(btc.TB(), err) @@ -416,14 +481,11 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { btcr.docsLock.Lock() defer btcr.docsLock.Unlock() - if _, ok := btcr.docs[docID]; ok { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID][revID] = bodyMessagePair + if doc, ok := btcr.docs[docID]; ok { + doc.addRevision(revID, body, msg) } else { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} + btcr.docs[docID] = btcr.NewBlipTesterDoc(revID, body, msg) } - btcr.updateLastReplicatedRev(docID, revID) if !msg.NoReply() { response := msg.Response() @@ -472,6 +534,10 @@ func (btc *BlipTesterCollectionClient) TB() testing.TB { return btc.parent.rt.TB } +func (btcc *BlipTesterCollectionClient) UseHLV() bool { + return btcc.parent.UseHLV() +} + // saveAttachment takes a content-type, and base64 encoded data and stores the attachment on the client func (btc *BlipTesterCollectionClient) saveAttachment(_, base64data string) (dataLength int, digest string, err error) { btc.attachmentsLock.Lock() @@ -506,32 +572,6 @@ func (btc *BlipTesterCollectionClient) getAttachment(digest string) (attachment return attachment, nil } -func (btc *BlipTesterCollectionClient) updateLastReplicatedRev(docID, revID string) { - btc.lastReplicatedRevLock.Lock() - defer btc.lastReplicatedRevLock.Unlock() - - currentRevID, ok := btc.lastReplicatedRev[docID] - if !ok { - btc.lastReplicatedRev[docID] = revID - return - } - - ctx := base.TestCtx(btc.parent.rt.TB) - currentGen, _ := db.ParseRevID(ctx, currentRevID) - incomingGen, _ := db.ParseRevID(ctx, revID) - if incomingGen > currentGen { - btc.lastReplicatedRev[docID] = revID - } -} - -func (btc *BlipTesterCollectionClient) getLastReplicatedRev(docID string) (revID string, ok bool) { - btc.lastReplicatedRevLock.RLock() - defer btc.lastReplicatedRevLock.RUnlock() - - revID, ok = btc.lastReplicatedRev[docID] - return revID, ok -} - func newBlipTesterReplication(tb testing.TB, id string, btc *BlipTesterClient, skipCollectionsInitialization bool) (*BlipTesterReplicator, error) { bt, err := NewBlipTesterFromSpecWithRT(tb, &BlipTesterSpec{ connectingPassword: "test", @@ -558,9 +598,9 @@ func newBlipTesterReplication(tb testing.TB, id string, btc *BlipTesterClient, s // getCollectionsForBLIP returns collections configured by a single database instance on a restTester. If only default collection exists, it will skip returning it to test "legacy" blip mode. func getCollectionsForBLIP(_ testing.TB, rt *RestTester) []string { - db := rt.GetDatabase() + dbc := rt.GetDatabase() var collections []string - for _, collection := range db.CollectionByID { + for _, collection := range dbc.CollectionByID { if base.IsDefaultCollection(collection.ScopeName, collection.Name) { continue } @@ -659,10 +699,9 @@ func (btc *BlipTesterClient) createBlipTesterReplications() error { } } else { btc.nonCollectionAwareClient = &BlipTesterCollectionClient{ - docs: make(map[string]map[string]*BodyMessagePair), - attachments: make(map[string][]byte), - lastReplicatedRev: make(map[string]string), - parent: btc, + docs: make(map[string]*BlipTesterDoc), + attachments: make(map[string][]byte), + parent: btc, } } @@ -674,10 +713,9 @@ func (btc *BlipTesterClient) createBlipTesterReplications() error { func (btc *BlipTesterClient) initCollectionReplication(collection string, collectionIdx int) error { btcReplicator := &BlipTesterCollectionClient{ - docs: make(map[string]map[string]*BodyMessagePair), - attachments: make(map[string][]byte), - lastReplicatedRev: make(map[string]string), - parent: btc, + docs: make(map[string]*BlipTesterDoc), + attachments: make(map[string][]byte), + parent: btc, } btcReplicator.collection = collection @@ -790,13 +828,9 @@ func (btc *BlipTesterCollectionClient) UnsubPushChanges() (response []byte, err // Close will empty the stored docs and close the underlying replications. func (btc *BlipTesterCollectionClient) Close() { btc.docsLock.Lock() - btc.docs = make(map[string]map[string]*BodyMessagePair, 0) + btc.docs = make(map[string]*BlipTesterDoc, 0) btc.docsLock.Unlock() - btc.lastReplicatedRevLock.Lock() - btc.lastReplicatedRev = make(map[string]string, 0) - btc.lastReplicatedRevLock.Unlock() - btc.attachmentsLock.Lock() btc.attachments = make(map[string][]byte, 0) btc.attachmentsLock.Unlock() @@ -814,20 +848,66 @@ func (btr *BlipTesterReplicator) sendMsg(msg *blip.Message) (err error) { // PushRev creates a revision on the client, and immediately sends a changes request for it. // The rev ID is always: "N-abc", where N is rev generation for predictability. func (btc *BlipTesterCollectionClient) PushRev(docID string, parentVersion DocVersion, body []byte) (DocVersion, error) { - revid, err := btc.PushRevWithHistory(docID, parentVersion.RevID, body, 1, 0) - return DocVersion{RevID: revid}, err + revid, err := btc.PushRevWithHistory(docID, parentVersion.RevTreeID, body, 1, 0) + if err != nil { + return DocVersion{}, err + } + docVersion := btc.GetDocVersion(docID) + require.Equal(btc.parent.rt.TB, docVersion.RevTreeID, revid) + return docVersion, nil +} + +// GetDocVersion fetches revid and cv directly from the bucket. Used to support REST-based verification in btc tests +// even while REST only supports revTreeId +func (btc *BlipTesterCollectionClient) GetDocVersion(docID string) DocVersion { + + collection := btc.parent.rt.GetSingleTestDatabaseCollection() + ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB), btc.parent.rt.GetDatabase().Name, nil) + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalSync) + require.NoError(btc.parent.rt.TB, err) + if doc.HLV == nil { + return DocVersion{RevTreeID: doc.CurrentRev} + } + return DocVersion{RevTreeID: doc.CurrentRev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } // PushRevWithHistory creates a revision on the client with history, and immediately sends a changes request for it. func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev string, body []byte, revCount, prunedRevCount int) (revID string, err error) { ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB), btc.parent.rt.GetDatabase().Name, nil) - parentRevGen, _ := db.ParseRevID(ctx, parentRev) - revGen := parentRevGen + revCount + prunedRevCount + revGen := 0 + newRevID := "" var revisionHistory []string - for i := revGen - 1; i > parentRevGen; i-- { - rev := fmt.Sprintf("%d-%s", i, "abc") - revisionHistory = append(revisionHistory, rev) + if btc.UseHLV() { + // When using version vectors: + // - source is "abc" + // - version value is simple counter + // - revisionHistory is just previous cv (parentRev) for changes response + startValue := uint64(0) + if parentRev != "" { + parentVersion, _ := db.ParseDecodedVersion(parentRev) + startValue = parentVersion.Value + revisionHistory = append(revisionHistory, parentRev) + } + newVersion := db.DecodedVersion{SourceID: "abc", Value: startValue + uint64(revCount) + 1} + newRevID = newVersion.String() + + } else { + // When using revtrees: + // - all revIDs are of the form [generation]-abc + // - [revCount] history entries are generated between the parent and the new rev + parentRevGen, _ := db.ParseRevID(ctx, parentRev) + revGen = parentRevGen + revCount + prunedRevCount + + for i := revGen - 1; i > parentRevGen; i-- { + rev := fmt.Sprintf("%d-%s", i, "abc") + revisionHistory = append(revisionHistory, rev) + } + if parentRev != "" { + + revisionHistory = append(revisionHistory, parentRev) + } + newRevID = fmt.Sprintf("%d-%s", revGen, "abc") } // Inline attachment processing @@ -837,19 +917,16 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin } var parentDocBody []byte - newRevID := fmt.Sprintf("%d-%s", revGen, "abc") btc.docsLock.Lock() if parentRev != "" { - revisionHistory = append(revisionHistory, parentRev) - if _, ok := btc.docs[docID]; ok { - // create new rev if doc and parent rev already exists - if parentDoc, okParent := btc.docs[docID][parentRev]; okParent { - parentDocBody = parentDoc.body - bodyMessagePair := &BodyMessagePair{body: body} - btc.docs[docID][newRevID] = bodyMessagePair + if doc, ok := btc.docs[docID]; ok { + // create new rev if doc exists and parent rev is current rev + if doc.getCurrentRevID() == parentRev { + parentDocBody = doc.body + doc.addRevision(newRevID, body, nil) } else { btc.docsLock.Unlock() - return "", fmt.Errorf("docID: %v with parent rev: %v was not found on the client", docID, parentRev) + return "", fmt.Errorf("docID: %v with current rev: %v was not found on the client", docID, parentRev) } } else { btc.docsLock.Unlock() @@ -858,8 +935,7 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin } else { // create new doc + rev if _, ok := btc.docs[docID]; !ok { - bodyMessagePair := &BodyMessagePair{body: body} - btc.docs[docID] = map[string]*BodyMessagePair{newRevID: bodyMessagePair} + btc.docs[docID] = btc.NewBlipTesterDoc(newRevID, body, nil) } } btc.docsLock.Unlock() @@ -936,7 +1012,6 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin return "", fmt.Errorf("error %s %s from revResponse: %s", revResponse.Properties["Error-Domain"], revResponse.Properties["Error-Code"], rspBody) } - btc.updateLastReplicatedRev(docID, newRevID) return newRevID, nil } @@ -947,8 +1022,9 @@ func (btc *BlipTesterCollectionClient) StoreRevOnClient(docID, revID string, bod if err != nil { return err } - bodyMessagePair := &BodyMessagePair{body: newBody} - btc.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} + btc.docsLock.Lock() + defer btc.docsLock.Unlock() + btc.docs[docID] = btc.NewBlipTesterDoc(revID, newBody, nil) return nil } @@ -1004,22 +1080,27 @@ func (btc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte return inputBody, nil } -// GetVersion returns the data stored in the Client under the given docID and version -func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVersion) (data []byte, found bool) { +// GetCurrentRevID gets the current revID for the specified docID +func (btc *BlipTesterCollectionClient) GetCurrentRevID(docID string) (revID string, data []byte, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - if data, ok := rev[docVersion.RevID]; ok && data != nil { - return data.body, true - } - // lookup by cv if not found using revid - if data, ok := rev[docVersion.CV.String()]; ok && data != nil { - return data.body, true - } + if doc, ok := btc.docs[docID]; ok { + return doc.getCurrentRevID(), doc.body, true } - return nil, false + return "", nil, false +} + +func (btc *BlipTesterClient) UseHLV() bool { + for _, protocol := range btc.SupportedBLIPProtocols { + subProtocol, err := db.ParseSubprotocolString(protocol) + require.NoError(btc.rt.TB, err) + if subProtocol >= db.CBMobileReplicationV4 { + return true + } + } + return false } func (btc *BlipTesterClient) AssertOnBlipHistory(t *testing.T, msg *blip.Message, docVersion DocVersion) { @@ -1030,8 +1111,27 @@ func (btc *BlipTesterClient) AssertOnBlipHistory(t *testing.T, msg *blip.Message assert.Equal(t, docVersion.CV.String(), msg.Properties[db.RevMessageHistory]) } } else { - assert.Equal(t, docVersion.RevID, msg.Properties[db.RevMessageHistory]) + assert.Equal(t, docVersion.RevTreeID, msg.Properties[db.RevMessageHistory]) + } +} + +// GetVersion returns the document body when the provided version matches the document's current revision +func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVersion) (data []byte, found bool) { + btc.docsLock.RLock() + defer btc.docsLock.RUnlock() + + if doc, ok := btc.docs[docID]; ok { + if doc.revMode == revModeHLV { + if doc.getCurrentRevID() == docVersion.CV.String() { + return doc.body, true + } + } else { + if doc.getCurrentRevID() == docVersion.RevTreeID { + return doc.body, true + } + } } + return nil, false } // WaitForVersion blocks until the given document version has been stored by the client, and returns the data when found. The test will fail after 10 seocnds if a matching document is not found. @@ -1047,15 +1147,13 @@ func (btc *BlipTesterCollectionClient) WaitForVersion(docID string, docVersion D return data } -// GetDoc returns a rev stored in the Client under the given docID. (if multiple revs are present, rev body returned is non-deterministic) +// GetDoc returns the current body stored in the Client for the given docID. func (btc *BlipTesterCollectionClient) GetDoc(docID string) (data []byte, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - for _, data := range rev { - return data.body, true - } + if doc, ok := btc.docs[docID]; ok { + return doc.body, true } return nil, false @@ -1119,28 +1217,29 @@ func (btr *BlipTesterReplicator) storeMessage(msg *blip.Message) { } // WaitForBlipRevMessage blocks until the given doc ID and rev ID has been stored by the client, and returns the message when found. If not found after 10 seconds, test will fail. -func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, docVersion DocVersion) (msg *blip.Message) { +func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, version DocVersion) (msg *blip.Message) { + var revID string + if btc.UseHLV() { + revID = version.CV.String() + } else { + revID = version.RevTreeID + } + require.EventuallyWithT(btc.TB(), func(c *assert.CollectT) { var ok bool - msg, ok = btc.GetBlipRevMessage(docID, docVersion) - assert.True(c, ok, "Could not find docID:%+v, RevID: %+v", docID, docVersion.RevID) - }, 10*time.Second, 50*time.Millisecond, "BlipTesterReplicator timed out waiting for BLIP message") + msg, ok = btc.GetBlipRevMessage(docID, revID) + assert.True(c, ok, "Could not find docID:%+v, RevID: %+v", docID, revID) + }, 10*time.Second, 50*time.Millisecond, "BlipTesterClient timed out waiting for BLIP message docID: %v, revID: %v", docID, revID) return msg } -func (btc *BlipTesterCollectionClient) GetBlipRevMessage(docID string, version DocVersion) (msg *blip.Message, found bool) { +func (btc *BlipTesterCollectionClient) GetBlipRevMessage(docID string, revID string) (msg *blip.Message, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - if pair, found := rev[version.RevID]; found { - found = pair.message != nil - return pair.message, found - } - // lookup by cv if not found using revid - if pair, found := rev[version.CV.String()]; found { - found = pair.message != nil - return pair.message, found + if doc, ok := btc.docs[docID]; ok { + if message, found := doc.revMessageHistory[revID]; found { + return message, found } } @@ -1162,8 +1261,8 @@ func (btcRunner *BlipTestClientRunner) WaitForDoc(clientID uint32, docID string) } // WaitForBlipRevMessage blocks until the given doc ID and rev ID has been stored by the client, and returns the message when found. If document is not found after 10 seconds, test will fail. -func (btcRunner *BlipTestClientRunner) WaitForBlipRevMessage(clientID uint32, docID string, docVersion DocVersion) *blip.Message { - return btcRunner.SingleCollection(clientID).WaitForBlipRevMessage(docID, docVersion) +func (btcRunner *BlipTestClientRunner) WaitForBlipRevMessage(clientID uint32, docID string, version DocVersion) *blip.Message { + return btcRunner.SingleCollection(clientID).WaitForBlipRevMessage(docID, version) } func (btcRunner *BlipTestClientRunner) StartOneshotPull(clientID uint32) error { @@ -1190,8 +1289,8 @@ func (btcRunner *BlipTestClientRunner) StartFilteredPullSince(clientID uint32, c return btcRunner.SingleCollection(clientID).StartPullSince(continuous, since, activeOnly, channels, "") } -func (btcRunner *BlipTestClientRunner) GetVersion(clientID uint32, docID string, docVersion DocVersion) ([]byte, bool) { - return btcRunner.SingleCollection(clientID).GetVersion(docID, docVersion) +func (btcRunner *BlipTestClientRunner) GetVersion(clientID uint32, docID string, version DocVersion) ([]byte, bool) { + return btcRunner.SingleCollection(clientID).GetVersion(docID, version) } func (btcRunner *BlipTestClientRunner) saveAttachment(clientID uint32, contentType string, attachmentData string) (int, string, error) { diff --git a/rest/bulk_api.go b/rest/bulk_api.go index 15f6b731a6..2499052226 100644 --- a/rest/bulk_api.go +++ b/rest/bulk_api.go @@ -142,7 +142,7 @@ func (h *handler) handleAllDocs() error { row.Status = http.StatusForbidden return row } - // handle the case where the incoming doc.RevID == "" + // handle the case where the incoming doc.RevTreeID == "" // and Get1xRevAndChannels returns the current revision doc.RevID = currentRevID } diff --git a/rest/changes_test.go b/rest/changes_test.go index f06e90e4ab..f1806ee6e4 100644 --- a/rest/changes_test.go +++ b/rest/changes_test.go @@ -238,7 +238,7 @@ func TestWebhookWinningRevChangedEvent(t *testing.T) { // push winning branch wg.Add(2) - res := rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"buzz","_revisions":{"start":3,"ids":["buzz","bar","`+version1.RevID+`"]}}`) + res := rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"buzz","_revisions":{"start":3,"ids":["buzz","bar","`+version1.RevTreeID+`"]}}`) RequireStatus(t, res, http.StatusCreated) winningVersion := DocVersionFromPutResponse(t, res) @@ -261,7 +261,7 @@ func TestWebhookWinningRevChangedEvent(t *testing.T) { // push a separate winning branch wg.Add(2) - res = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"quux","_revisions":{"start":4,"ids":["quux", "buzz","bar","`+version1.RevID+`"]}}`) + res = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"quux","_revisions":{"start":4,"ids":["quux", "buzz","bar","`+version1.RevTreeID+`"]}}`) RequireStatus(t, res, http.StatusCreated) newWinningVersion := DocVersionFromPutResponse(t, res) diff --git a/rest/changestest/changes_api_test.go b/rest/changestest/changes_api_test.go index e94c1913e6..d91dd1325f 100644 --- a/rest/changestest/changes_api_test.go +++ b/rest/changestest/changes_api_test.go @@ -807,10 +807,10 @@ func TestPostChangesAdminChannelGrantRemovalWithLimit(t *testing.T) { cacheWaiter.AddAndWait(4) // Mark the first four PBS docs as removals - _ = rt.PutDoc("pbs-1", fmt.Sprintf(`{"_rev":%q}`, pbs1.RevID)) - _ = rt.PutDoc("pbs-2", fmt.Sprintf(`{"_rev":%q}`, pbs2.RevID)) - _ = rt.PutDoc("pbs-3", fmt.Sprintf(`{"_rev":%q}`, pbs3.RevID)) - _ = rt.PutDoc("pbs-4", fmt.Sprintf(`{"_rev":%q}`, pbs4.RevID)) + _ = rt.PutDoc("pbs-1", fmt.Sprintf(`{"_rev":%q}`, pbs1.RevTreeID)) + _ = rt.PutDoc("pbs-2", fmt.Sprintf(`{"_rev":%q}`, pbs2.RevTreeID)) + _ = rt.PutDoc("pbs-3", fmt.Sprintf(`{"_rev":%q}`, pbs3.RevTreeID)) + _ = rt.PutDoc("pbs-4", fmt.Sprintf(`{"_rev":%q}`, pbs4.RevTreeID)) cacheWaiter.AddAndWait(4) @@ -886,7 +886,7 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { cacheWaiter.AddAndWait(4) // remove channels/tombstone a couple of docs to ensure they're not backfilled after a dynamic grant - _ = rt.PutDoc("hbo-2", fmt.Sprintf(`{"_rev":%q}`, hbo2.RevID)) + _ = rt.PutDoc("hbo-2", fmt.Sprintf(`{"_rev":%q}`, hbo2.RevTreeID)) rt.DeleteDoc(pbs2ID, pbs2Version) cacheWaiter.AddAndWait(2) diff --git a/rest/importtest/import_test.go b/rest/importtest/import_test.go index 56c4b68641..5355adc97d 100644 --- a/rest/importtest/import_test.go +++ b/rest/importtest/import_test.go @@ -140,7 +140,7 @@ func TestXattrImportOldDocRevHistory(t *testing.T) { // 1. Create revision with history docID := t.Name() version := rt.PutDoc(docID, `{"val":-1}`) - revID := version.RevID + revID := version.RevTreeID collection := rt.GetSingleTestDatabaseCollectionWithUser() ctx := rt.Context() @@ -149,7 +149,7 @@ func TestXattrImportOldDocRevHistory(t *testing.T) { // Purge old revision JSON to simulate expiry, and to verify import doesn't attempt multiple retrievals purgeErr := collection.PurgeOldRevisionJSON(ctx, docID, revID) require.NoError(t, purgeErr) - revID = version.RevID + revID = version.RevTreeID } // 2. Modify doc via SDK diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index 25aaf63e92..1ddfa725bf 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -2832,7 +2832,7 @@ func TestActiveReplicatorPullMergeConflictingAttachments(t *testing.T) { rt1.WaitForReplicationStatus("repl1", db.ReplicationStateStopped) - resp = rt1.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevID, test.localConflictingRevBody) + resp = rt1.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevTreeID, test.localConflictingRevBody) rest.RequireStatus(t, resp, http.StatusCreated) changesResults, err = rt1.WaitForChanges(1, "/{{.keyspace}}/_changes?since="+lastSeq, "", true) @@ -2841,7 +2841,7 @@ func TestActiveReplicatorPullMergeConflictingAttachments(t *testing.T) { assert.Equal(t, docID, changesResults.Results[0].ID) lastSeq = changesResults.Last_Seq.(string) - resp = rt2.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevID, test.remoteConflictingRevBody) + resp = rt2.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevTreeID, test.remoteConflictingRevBody) rest.RequireStatus(t, resp, http.StatusCreated) resp = rt1.SendAdminRequest(http.MethodPut, "/{{.db}}/_replicationStatus/repl1?action=start", "") @@ -5905,7 +5905,7 @@ func TestActiveReplicatorPullConflictReadWriteIntlProps(t *testing.T) { createVersion := func(generation int, parentRevID string, body db.Body) rest.DocVersion { rev, err := db.CreateRevID(generation, parentRevID, body) require.NoError(t, err, "Error creating revision") - return rest.DocVersion{RevID: rev} + return rest.DocVersion{RevTreeID: rev} } docExpiry := time.Now().Local().Add(time.Hour * time.Duration(4)).Format(time.RFC3339) @@ -6346,7 +6346,7 @@ func TestSGR2TombstoneConflictHandling(t *testing.T) { assert.NoError(t, err) // Create another rev and then delete doc on local - ie tree is longer - version := localActiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevID: "3-abc"}, `{"foo":"bar"}`) + version := localActiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevTreeID: "3-abc"}, `{"foo":"bar"}`) localActiveRT.DeleteDoc(doc2ID, version) // Validate local is CBS tombstone, expect not found error @@ -6372,7 +6372,7 @@ func TestSGR2TombstoneConflictHandling(t *testing.T) { assert.NoError(t, err) // Create another rev and then delete doc on remotePassiveRT (passive) - ie, tree is longer - version := remotePassiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevID: "3-abc"}, `{"foo":"bar"}`) + version := remotePassiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevTreeID: "3-abc"}, `{"foo":"bar"}`) remotePassiveRT.DeleteDoc(doc2ID, version) // Validate local is CBS tombstone, expect not found error @@ -7325,7 +7325,7 @@ func TestReplicatorDoNotSendDeltaWhenSrcIsTombstone(t *testing.T) { // Replicate tombstone to passive err = passiveRT.WaitForCondition(func() bool { - rawResponse := passiveRT.SendAdminRequest("GET", "/{{.keyspace}}/test?rev="+deletedVersion.RevID, "") + rawResponse := passiveRT.SendAdminRequest("GET", "/{{.keyspace}}/test?rev="+deletedVersion.RevTreeID, "") return rawResponse.Code == 404 }) require.NoError(t, err) @@ -7489,7 +7489,7 @@ func TestReplicatorIgnoreRemovalBodies(t *testing.T) { require.NoError(t, activeRT.WaitForVersion(docID, version3)) activeRT.GetSingleTestDatabaseCollection().FlushRevisionCacheForTest() - err := activeRT.GetSingleDataStore().Delete(fmt.Sprintf("_sync:rev:%s:%d:%s", t.Name(), len(version2.RevID), version2.RevID)) + err := activeRT.GetSingleDataStore().Delete(fmt.Sprintf("_sync:rev:%s:%d:%s", t.Name(), len(version2.RevTreeID), version2.RevTreeID)) require.NoError(t, err) // Set-up replicator // passiveDBURL, err := url.Parse(srv.URL + "/db") diff --git a/rest/replicatortest/replicator_test_helper.go b/rest/replicatortest/replicator_test_helper.go index e95ddb347c..34bf99c2f9 100644 --- a/rest/replicatortest/replicator_test_helper.go +++ b/rest/replicatortest/replicator_test_helper.go @@ -71,7 +71,7 @@ func addActiveRT(t *testing.T, dbName string, testBucket *base.TestBucket) (acti // requireDocumentVersion asserts that the given ChangeRev has the expected version for a given entry returned by _changes feed func requireDocumentVersion(t testing.TB, expected rest.DocVersion, doc *db.Document) { - rest.RequireDocVersionEqual(t, expected, rest.DocVersion{RevID: doc.SyncData.CurrentRev}) + rest.RequireDocVersionEqual(t, expected, rest.DocVersion{RevTreeID: doc.SyncData.CurrentRev}) } // requireRevID asserts that the specified document version is written to the diff --git a/rest/revocation_test.go b/rest/revocation_test.go index 17095ff0f0..5f928dace0 100644 --- a/rest/revocation_test.go +++ b/rest/revocation_test.go @@ -1014,10 +1014,10 @@ func TestRevocationResumeAndLowSeqCheck(t *testing.T) { changes = revocationTester.getChanges(changes.Last_Seq, 2) assert.Equal(t, doc1ID, changes.Results[0].ID) - assert.Equal(t, doc1Version.RevID, changes.Results[0].Changes[0]["rev"]) + assert.Equal(t, doc1Version.RevTreeID, changes.Results[0].Changes[0]["rev"]) assert.True(t, changes.Results[0].Revoked) assert.Equal(t, doc2ID, changes.Results[1].ID) - assert.Equal(t, doc2Version.RevID, changes.Results[1].Changes[0]["rev"]) + assert.Equal(t, doc2Version.RevTreeID, changes.Results[1].Changes[0]["rev"]) assert.True(t, changes.Results[1].Revoked) changes = revocationTester.getChanges("20:40", 1) diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 26c7189e28..c9987e9c29 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -771,7 +771,7 @@ func (cr ChangesResults) RequireDocIDs(t testing.TB, docIDs []string) { // RequireChangeRevVersion asserts that the given ChangeRev has the expected version for a given entry returned by _changes feed func RequireChangeRevVersion(t *testing.T, expected DocVersion, changeRev db.ChangeRev) { - RequireDocVersionEqual(t, expected, DocVersion{RevID: changeRev["rev"]}) + RequireDocVersionEqual(t, expected, DocVersion{RevTreeID: changeRev["rev"]}) } func (rt *RestTester) CreateWaitForChangesRetryWorker(numChangesExpected int, changesURL, username string, useAdminPort bool) (worker base.RetryWorker) { @@ -2344,16 +2344,16 @@ func WaitAndAssertBackgroundManagerExpiredHeartbeat(t testing.TB, bm *db.Backgro // DocVersion represents a specific version of a document in an revID/HLV agnostic manner. type DocVersion struct { - RevID string - CV db.Version + RevTreeID string + CV db.Version } func (v *DocVersion) String() string { - return fmt.Sprintf("RevID: %s", v.RevID) + return fmt.Sprintf("RevTreeID: %s", v.RevTreeID) } func (v DocVersion) Equal(o DocVersion) bool { - if v.RevID != o.RevID { + if v.RevTreeID != o.RevTreeID { return false } return true @@ -2361,12 +2361,12 @@ func (v DocVersion) Equal(o DocVersion) bool { // Digest returns the digest for the current version func (v DocVersion) Digest() string { - return strings.Split(v.RevID, "-")[1] + return strings.Split(v.RevTreeID, "-")[1] } // RequireDocVersionNotNil calls t.Fail if two document version is not specified. func RequireDocVersionNotNil(t *testing.T, version DocVersion) { - require.NotEqual(t, "", version.RevID) + require.NotEqual(t, "", version.RevTreeID) } // RequireDocVersionEqual calls t.Fail if two document versions are not equal. @@ -2381,12 +2381,12 @@ func RequireDocVersionNotEqual(t *testing.T, expected, actual DocVersion) { // EmptyDocVersion reprents an empty document version. func EmptyDocVersion() DocVersion { - return DocVersion{RevID: ""} + return DocVersion{RevTreeID: ""} } // NewDocVersionFromFakeRev returns a new DocVersion from the given fake rev ID, intended for use when we explicit create conflicts. func NewDocVersionFromFakeRev(fakeRev string) DocVersion { - return DocVersion{RevID: fakeRev} + return DocVersion{RevTreeID: fakeRev} } // DocVersionFromPutResponse returns a DocRevisionID from the given response to PUT /{, or fails the given test if a rev ID was not found. @@ -2398,7 +2398,7 @@ func DocVersionFromPutResponse(t testing.TB, response *TestResponse) DocVersion require.NoError(t, json.Unmarshal(response.BodyBytes(), &r)) require.NotNil(t, r.RevID, "expecting non-nil rev ID from response: %s", string(response.BodyBytes())) require.NotEqual(t, "", *r.RevID, "expecting non-empty rev ID from response: %s", string(response.BodyBytes())) - return DocVersion{RevID: *r.RevID} + return DocVersion{RevTreeID: *r.RevID} } func MarshalConfig(t *testing.T, config db.ReplicationConfig) string { diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 2c23905c3d..580dd58362 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -54,12 +54,12 @@ func (rt *RestTester) GetDoc(docID string) (DocVersion, db.Body) { RevID *string `json:"_rev"` } require.NoError(rt.TB, base.JSONUnmarshal(rawResponse.Body.Bytes(), &r)) - return DocVersion{RevID: *r.RevID}, body + return DocVersion{RevTreeID: *r.RevID}, body } // GetDocVersion returns the doc body and version for the given docID and version. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocVersion(docID string, version DocVersion) db.Body { - rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+version.RevID, "") + rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+version.RevTreeID, "") RequireStatus(rt.TB, rawResponse, http.StatusOK) var body db.Body require.NoError(rt.TB, base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) @@ -82,13 +82,13 @@ func (rt *RestTester) PutDoc(docID string, body string) DocVersion { // UpdateDocRev updates a document at a specific revision and returns the new version. Deprecated for UpdateDoc. func (rt *RestTester) UpdateDocRev(docID, revID string, body string) string { - version := rt.UpdateDoc(docID, DocVersion{RevID: revID}, body) - return version.RevID + version := rt.UpdateDoc(docID, DocVersion{RevTreeID: revID}, body) + return version.RevTreeID } // UpdateDoc updates a document at a specific version and returns the new version. func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) DocVersion { - resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevID) + resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevTreeID) rawResponse := rt.SendAdminRequest(http.MethodPut, resource, body) RequireStatus(rt.TB, rawResponse, http.StatusCreated) return DocVersionFromPutResponse(rt.TB, rawResponse) @@ -102,14 +102,14 @@ func (rt *RestTester) DeleteDoc(docID string, docVersion DocVersion) { // DeleteDocReturnVersion deletes a document at a specific version. The test will fail if the revision does not exist. func (rt *RestTester) DeleteDocReturnVersion(docID string, docVersion DocVersion) DocVersion { resp := rt.SendAdminRequest(http.MethodDelete, - fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, docVersion.RevID), "") + fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, docVersion.RevTreeID), "") RequireStatus(rt.TB, resp, http.StatusOK) return DocVersionFromPutResponse(rt.TB, resp) } // DeleteDocRev removes a document at a specific revision. Deprecated for DeleteDoc. func (rt *RestTester) DeleteDocRev(docID, revID string) { - rt.DeleteDoc(docID, DocVersion{RevID: revID}) + rt.DeleteDoc(docID, DocVersion{RevTreeID: revID}) } func (rt *RestTester) GetDatabaseRoot(dbname string) DatabaseRoot { @@ -122,7 +122,7 @@ func (rt *RestTester) GetDatabaseRoot(dbname string) DatabaseRoot { // WaitForVersion retries a GET for a given document version until it returns 200 or 201 for a given document and revision. If version is not found, the test will fail. func (rt *RestTester) WaitForVersion(docID string, version DocVersion) error { - require.NotEqual(rt.TB, "", version.RevID) + require.NotEqual(rt.TB, "", version.RevTreeID) return rt.WaitForCondition(func() bool { rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID, "") if rawResponse.Code != 200 && rawResponse.Code != 201 { @@ -130,13 +130,13 @@ func (rt *RestTester) WaitForVersion(docID string, version DocVersion) error { } var body db.Body require.NoError(rt.TB, base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) - return body.ExtractRev() == version.RevID + return body.ExtractRev() == version.RevTreeID }) } // WaitForRev retries a GET until it returns 200 or 201. If revision is not found, the test will fail. This function is deprecated for RestTester.WaitForVersion func (rt *RestTester) WaitForRev(docID, revID string) error { - return rt.WaitForVersion(docID, DocVersion{RevID: revID}) + return rt.WaitForVersion(docID, DocVersion{RevTreeID: revID}) } func (rt *RestTester) WaitForCheckpointLastSequence(expectedName string) (string, error) { @@ -416,23 +416,23 @@ func (rt *RestTester) PutDocDirectly(docID string, body db.Body) DocVersion { collection := rt.GetSingleTestDatabaseCollectionWithUser() rev, doc, err := collection.Put(rt.Context(), docID, body) require.NoError(rt.TB, err) - return DocVersion{RevID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } func (rt *RestTester) UpdateDocDirectly(docID string, version DocVersion, body db.Body) DocVersion { collection := rt.GetSingleTestDatabaseCollectionWithUser() body[db.BodyId] = docID - body[db.BodyRev] = version.RevID + body[db.BodyRev] = version.RevTreeID rev, doc, err := collection.Put(rt.Context(), docID, body) require.NoError(rt.TB, err) - return DocVersion{RevID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } func (rt *RestTester) DeleteDocDirectly(docID string, version DocVersion) DocVersion { collection := rt.GetSingleTestDatabaseCollectionWithUser() - rev, doc, err := collection.DeleteDoc(rt.Context(), docID, version.RevID) + rev, doc, err := collection.DeleteDoc(rt.Context(), docID, version.RevTreeID) require.NoError(rt.TB, err) - return DocVersion{RevID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } func (rt *RestTester) PutDocDirectlyInCollection(collection *db.DatabaseCollection, docID string, body db.Body) DocVersion { @@ -441,5 +441,5 @@ func (rt *RestTester) PutDocDirectlyInCollection(collection *db.DatabaseCollecti } rev, doc, err := dbUser.Put(rt.Context(), docID, body) require.NoError(rt.TB, err) - return DocVersion{RevID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} }