From 37d3cb558225940936ca75d7ca1076a4129b8c00 Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Wed, 13 Mar 2024 06:40:21 -0700 Subject: [PATCH] CBG-3797 Attachment handling for HLV push replication (#6702) HLV clients don't consider revpos, and evaluate whether they need to request an attachment based on the existing set of attachments on the document. SGW still needs to persist revpos into _attachments to support revtree clients. For new attachments added by HLV client, revpos is set to the generation of SGW's computed revTreeID for the incoming revision. Co-authored-by: Gregory Newman-Smith --- db/blip_handler.go | 64 ++++++++++++--------- db/revision_cache_interface.go | 2 +- db/util_testing.go | 2 +- rest/attachment_test.go | 23 +++----- rest/blip_api_attachment_test.go | 97 +++++++++++++++++++------------- rest/blip_api_crud_test.go | 1 - rest/blip_client_test.go | 59 +++++++++++++++---- rest/utilities_testing.go | 11 ++++ 8 files changed, 164 insertions(+), 95 deletions(-) diff --git a/db/blip_handler.go b/db/blip_handler.go index 01dbe5b29a..9074ea7e14 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -808,7 +808,7 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error { } var status ProposedRevStatus var currentRev string - if bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 { + if bh.useHLV() { status, currentRev = bh.collection.CheckProposedVersion(bh.loggingCtx, docID, rev, parentRevID) } else { status, currentRev = bh.collection.CheckProposedRev(bh.loggingCtx, docID, rev, parentRevID) @@ -954,7 +954,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } }() - if bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 && bh.conflictResolver != nil { + if bh.useHLV() && bh.conflictResolver != nil { return base.HTTPErrorf(http.StatusNotImplemented, "conflict resolver handling (ISGR) not yet implemented for v4 protocol") } @@ -1026,17 +1026,18 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } var history []string + historyStr := rq.Properties[RevMessageHistory] var incomingHLV HybridLogicalVector // Build history/HLV - if bh.activeCBMobileSubprotocol < CBMobileReplicationV4 { + if !bh.useHLV() { newDoc.RevID = rev history = []string{rev} - if historyStr := rq.Properties[RevMessageHistory]; historyStr != "" { + if historyStr != "" { history = append(history, strings.Split(historyStr, ",")...) } } else { versionVectorStr := rev - if historyStr := rq.Properties[RevMessageHistory]; historyStr != "" { + if historyStr != "" { versionVectorStr += ";" + historyStr } incomingHLV, err = extractHLVFromBlipMessage(versionVectorStr) @@ -1066,7 +1067,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // due to no-conflict write restriction, but we still need to enforce security here to prevent leaking data about previous // revisions to malicious actors (in the scenario where that user has write but not read access). var deltaSrcRev DocumentRevision - if bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 { + if bh.useHLV() { cv := Version{} cv.SourceID, cv.Value = incomingHLV.GetCurrentVersion() deltaSrcRev, err = bh.collection.GetCV(bh.loggingCtx, docID, &cv, RevCacheOmitBody) @@ -1138,31 +1139,32 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err var rawBucketDoc *sgbucket.BucketDocument - // Pull out attachments + // Attachment processing if injectedAttachmentsForDelta || bytes.Contains(bodyBytes, []byte(BodyAttachments)) { - // temporarily error here if V4 - if bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 { - return base.HTTPErrorf(http.StatusNotImplemented, "attachment handling not yet supported for v4 protocol") - } + body := newDoc.Body(bh.loggingCtx) var currentBucketDoc *Document - // Look at attachments with revpos > the last common ancestor's - minRevpos := 1 - if len(history) > 0 { - currentDoc, rawDoc, err := bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, DocUnmarshalSync) - // If we're able to obtain current doc data then we should use the common ancestor generation++ for min revpos - // as we will already have any attachments on the common ancestor so don't need to ask for them. - // Otherwise we'll have to go as far back as we can in the doc history and choose the last entry in there. - if err == nil { - commonAncestor := currentDoc.History.findAncestorFromSet(currentDoc.CurrentRev, history) - minRevpos, _ = ParseRevID(bh.loggingCtx, commonAncestor) - minRevpos++ - rawBucketDoc = rawDoc - currentBucketDoc = currentDoc - } else { - minRevpos, _ = ParseRevID(bh.loggingCtx, history[len(history)-1]) + minRevpos := 0 + if historyStr != "" { + // fetch current bucket doc. Treats error as not found + currentBucketDoc, rawBucketDoc, _ = bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, DocUnmarshalSync) + + // For revtree clients, can use revPos as an optimization. HLV always compares incoming + // attachments with current attachments on the document + if !bh.useHLV() { + // Look at attachments with revpos > the last common ancestor's + // If we're able to obtain current doc data then we should use the common ancestor generation++ for min revpos + // as we will already have any attachments on the common ancestor so don't need to ask for them. + // Otherwise we'll have to go as far back as we can in the doc history and choose the last entry in there. + if currentBucketDoc != nil { + commonAncestor := currentBucketDoc.History.findAncestorFromSet(currentBucketDoc.CurrentRev, history) + minRevpos, _ = ParseRevID(bh.loggingCtx, commonAncestor) + minRevpos++ + } else { + minRevpos, _ = ParseRevID(bh.loggingCtx, history[len(history)-1]) + } } } @@ -1180,7 +1182,9 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err if !ok { // If we don't have this attachment already, ensure incoming revpos is greater than minRevPos, otherwise // update to ensure it's fetched and uploaded - bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, rev) + if minRevpos > 0 { + bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, rev) + } continue } @@ -1251,7 +1255,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // If the doc is a tombstone we want to allow conflicts when running SGR2 // bh.conflictResolver != nil represents an active SGR2 and BLIPClientTypeSGR2 represents a passive SGR2 forceAllowConflictingTombstone := newDoc.Deleted && (bh.conflictResolver != nil || bh.clientType == BLIPClientTypeSGR2) - if bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 { + if bh.useHLV() { _, _, _, err = bh.collection.PutExistingCurrentVersion(bh.loggingCtx, newDoc, incomingHLV, rawBucketDoc) } else if bh.conflictResolver != nil { _, _, err = bh.collection.PutExistingRevWithConflictResolution(bh.loggingCtx, newDoc, history, true, bh.conflictResolver, forceAllowConflictingTombstone, rawBucketDoc, ExistingVersionWithUpdateToHLV) @@ -1585,3 +1589,7 @@ func allowedAttachmentKey(docID, digest string, activeCBMobileSubprotocol CBMobi func (bh *blipHandler) logEndpointEntry(profile, endpoint string) { base.InfofCtx(bh.loggingCtx, base.KeySyncMsg, "#%d: Type:%s %s", bh.serialNumber, profile, endpoint) } + +func (bh *blipHandler) useHLV() bool { + return bh.activeCBMobileSubprotocol >= CBMobileReplicationV4 +} diff --git a/db/revision_cache_interface.go b/db/revision_cache_interface.go index e0277b9600..e1667f2fad 100644 --- a/db/revision_cache_interface.go +++ b/db/revision_cache_interface.go @@ -332,7 +332,7 @@ func revCacheLoaderForDocument(ctx context.Context, backingStore RevisionCacheBa func revCacheLoaderForDocumentCV(ctx context.Context, backingStore RevisionCacheBackingStore, doc *Document, cv Version) (bodyBytes []byte, body Body, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, revid string, hlv *HybridLogicalVector, err error) { if bodyBytes, body, attachments, err = backingStore.getCurrentVersion(ctx, doc); err != nil { // TODO: CBG-3814 - pending support of channel removal for CV - base.ErrorfCtx(ctx, "pending CBG-3213 support of channel removal for CV: %v", err) + base.ErrorfCtx(ctx, "pending CBG-3814 support of channel removal for CV: %v", err) } if err = doc.HasCurrentVersion(ctx, cv); err != nil { diff --git a/db/util_testing.go b/db/util_testing.go index f45129465b..fe68c14b67 100644 --- a/db/util_testing.go +++ b/db/util_testing.go @@ -682,7 +682,7 @@ func (c *DatabaseCollection) RequireCurrentVersion(t *testing.T, key string, sou } // GetDocumentCurrentVersion fetches the document by key and returns the current version -func (c *DatabaseCollection) GetDocumentCurrentVersion(t *testing.T, key string) (source string, version string) { +func (c *DatabaseCollection) GetDocumentCurrentVersion(t testing.TB, key string) (source string, version string) { ctx := base.TestCtx(t) doc, err := c.GetDocument(ctx, key, DocUnmarshalSync) require.NoError(t, err) diff --git a/rest/attachment_test.go b/rest/attachment_test.go index 67df6cb566..175d8bced0 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -2265,7 +2265,6 @@ func TestUpdateExistingAttachment(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // attachments not yet replicated in V4 protocol (CBG-3797) const ( doc1ID = "doc1" doc2ID = "doc2" @@ -2279,8 +2278,8 @@ func TestUpdateExistingAttachment(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() - doc1Version := rt.PutDoc(doc1ID, `{}`) - doc2Version := rt.PutDoc(doc2ID, `{}`) + doc1Version := btc.PutDoc(doc1ID, `{}`) + doc2Version := btc.PutDoc(doc2ID, `{}`) require.NoError(t, rt.WaitForPendingChanges()) @@ -2328,7 +2327,6 @@ func TestPushUnknownAttachmentAsStub(t *testing.T) { } const doc1ID = "doc1" btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // attachments not yet replicated in V4 protocol (CBG-3797) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -2338,7 +2336,7 @@ func TestPushUnknownAttachmentAsStub(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() // Add doc1 and doc2 - doc1Version := btc.rt.PutDoc(doc1ID, `{}`) + doc1Version := btc.PutDoc(doc1ID, `{}`) require.NoError(t, btc.rt.WaitForPendingChanges()) @@ -2378,7 +2376,6 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // attachments not yet replicated in V4 protocol (CBG-3797) const docID = "doc" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { @@ -2389,7 +2386,7 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() // Push an initial rev with attachment data - initialVersion := btc.rt.PutDoc(docID, `{"_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) + initialVersion := btc.PutDoc(docID, `{"_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) err := btc.rt.WaitForPendingChanges() assert.NoError(t, err) @@ -2401,7 +2398,7 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { // Push a revision with a bunch of history simulating doc updated on mobile device // Note this references revpos 1 and therefore SGW has it - Shouldn't need proveAttachment proveAttachmentBefore := btc.pushReplication.replicationStats.ProveAttachment.Value() - revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.RevTreeID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) + revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.GetRev(btc.UseHLV()), []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) assert.NoError(t, err) proveAttachmentAfter := btc.pushReplication.replicationStats.ProveAttachment.Value() assert.Equal(t, proveAttachmentBefore, proveAttachmentAfter) @@ -2420,7 +2417,6 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // attachments not yet replicated in V4 protocol (CBG-3797) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -2431,7 +2427,7 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { defer btc.Close() // Create rev 1 with the hello.txt attachment const docID = "doc" - version := btc.rt.PutDoc(docID, `{"val": "val", "_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) + version := btc.PutDoc(docID, `{"val": "val", "_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) err := btc.rt.WaitForPendingChanges() assert.NoError(t, err) @@ -2446,7 +2442,7 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { btcRunner.AttachmentsLock(btc.id).Unlock() // Put doc with an erroneous revpos 1 but with a different digest, referring to the above attachment - _, err = btcRunner.PushRevWithHistory(btc.id, docID, version.RevTreeID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) + _, err = btcRunner.PushRevWithHistory(btc.id, docID, version.GetRev(btc.UseHLV()), []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) require.NoError(t, err) // Ensure message and attachment is pushed up @@ -2602,7 +2598,6 @@ func TestCBLRevposHandling(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // attachments not yet replicated in V4 protocol (CBG-3797) const ( doc1ID = "doc1" doc2ID = "doc2" @@ -2616,8 +2611,8 @@ func TestCBLRevposHandling(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() - doc1Version := btc.rt.PutDoc(doc1ID, `{}`) - doc2Version := btc.rt.PutDoc(doc2ID, `{}`) + doc1Version := btc.PutDoc(doc1ID, `{}`) + doc2Version := btc.PutDoc(doc2ID, `{}`) require.NoError(t, btc.rt.WaitForPendingChanges()) err := btcRunner.StartOneshotPull(btc.id) diff --git a/rest/blip_api_attachment_test.go b/rest/blip_api_attachment_test.go index 6b9a5b486e..9449ae5d3a 100644 --- a/rest/blip_api_attachment_test.go +++ b/rest/blip_api_attachment_test.go @@ -290,8 +290,8 @@ func TestBlipPushPullNewAttachmentCommonAncestor(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) const docID = "doc1" + ctx := base.TestCtx(t) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -306,50 +306,58 @@ func TestBlipPushPullNewAttachmentCommonAncestor(t *testing.T) { // CBL creates revisions 1-abc,2-abc on the client, with an attachment associated with rev 2. bodyText := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` - err = btcRunner.StoreRevOnClient(btc.id, docID, "2-abc", []byte(bodyText)) + rev := "2-abc" + if btc.UseHLV() { + rev = db.EncodeTestVersion("2@abc") + } + err = btcRunner.StoreRevOnClient(btc.id, docID, rev, []byte(bodyText)) require.NoError(t, err) bodyText = `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` revId, err := btcRunner.PushRevWithHistory(btc.id, docID, "", []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "2-abc", revId) + assert.Equal(t, rev, revId) // Wait for the documents to be replicated at SG btc.pushReplication.WaitForMessage(2) - resp := btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) + collection := rt.GetSingleTestDatabaseCollection() + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + + attachmentRevPos, _ := db.ParseRevID(ctx, doc.CurrentRev) // CBL updates the doc w/ two more revisions, 3-abc, 4-abc, // these are sent to SG as 4-abc, history:[4-abc,3-abc,2-abc], the attachment has revpos=2 bodyText = `{"greetings":[{"hi":"bob"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` revId, err = btcRunner.PushRevWithHistory(btc.id, docID, revId, []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "4-abc", revId) + expectedRev := "4-abc" + if btc.UseHLV() { + expectedRev = db.EncodeTestVersion("4@abc") + } + assert.Equal(t, expectedRev, revId) // Wait for the document to be replicated at SG btc.pushReplication.WaitForMessage(4) - resp = btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) - - var respBody db.Body - assert.NoError(t, base.JSONUnmarshal(resp.Body.Bytes(), &respBody)) + doc, err = collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) - assert.Equal(t, docID, respBody[db.BodyId]) - assert.Equal(t, "4-abc", respBody[db.BodyRev]) - greetings := respBody["greetings"].([]interface{}) + btc.RequireRev(t, expectedRev, doc) + body := doc.Body(ctx) + greetings := body["greetings"].([]interface{}) assert.Len(t, greetings, 1) assert.Equal(t, map[string]interface{}{"hi": "bob"}, greetings[0]) - attachments, ok := respBody[db.BodyAttachments].(map[string]interface{}) - require.True(t, ok) - assert.Len(t, attachments, 1) - hello, ok := attachments["hello.txt"].(map[string]interface{}) + assert.Len(t, doc.Attachments, 1) + hello, ok := doc.Attachments["hello.txt"].(map[string]interface{}) require.True(t, ok) assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) assert.Equal(t, float64(11), hello["length"]) - assert.Equal(t, float64(2), hello["revpos"]) + + // revpos should mach the generation of the original revision + assert.Equal(t, float64(attachmentRevPos), hello["revpos"]) assert.True(t, hello["stub"].(bool)) // Check the number of sendProveAttachment/sendGetAttachment calls. @@ -366,7 +374,7 @@ func TestBlipPushPullNewAttachmentNoCommonAncestor(t *testing.T) { const docID = "doc1" btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) + ctx := base.TestCtx(t) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -382,37 +390,47 @@ func TestBlipPushPullNewAttachmentNoCommonAncestor(t *testing.T) { // rev tree pruning on the CBL side, so 1-abc no longer exists. // CBL replicates, sends to client as 4-abc history:[4-abc, 3-abc, 2-abc], attachment has revpos=2 bodyText := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` - err = btcRunner.StoreRevOnClient(btc.id, docID, "2-abc", []byte(bodyText)) + rev := "2-abc" + if btc.UseHLV() { + rev = db.EncodeTestVersion("2@abc") + } + err = btcRunner.StoreRevOnClient(btc.id, docID, rev, []byte(bodyText)) require.NoError(t, err) bodyText = `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` - revId, err := btcRunner.PushRevWithHistory(btc.id, docID, "2-abc", []byte(bodyText), 2, 0) + currentRev, err := btcRunner.PushRevWithHistory(btc.id, docID, rev, []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "4-abc", revId) + expectedRev := "4-abc" + if btc.UseHLV() { + expectedRev = db.EncodeTestVersion("4@abc") + } + assert.Equal(t, expectedRev, currentRev) // Wait for the document to be replicated at SG btc.pushReplication.WaitForMessage(2) - resp := btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) + collection := rt.GetSingleTestDatabaseCollection() + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) - var respBody db.Body - assert.NoError(t, base.JSONUnmarshal(resp.Body.Bytes(), &respBody)) + btc.RequireRev(t, expectedRev, doc) - assert.Equal(t, docID, respBody[db.BodyId]) - assert.Equal(t, "4-abc", respBody[db.BodyRev]) - greetings := respBody["greetings"].([]interface{}) + body := doc.Body(ctx) + greetings := body["greetings"].([]interface{}) assert.Len(t, greetings, 1) assert.Equal(t, map[string]interface{}{"hi": "alice"}, greetings[0]) - attachments, ok := respBody[db.BodyAttachments].(map[string]interface{}) - require.True(t, ok) - assert.Len(t, attachments, 1) - hello, ok := attachments["hello.txt"].(map[string]interface{}) + assert.Len(t, doc.Attachments, 1) + hello, ok := doc.Attachments["hello.txt"].(map[string]interface{}) require.True(t, ok) assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) assert.Equal(t, float64(11), hello["length"]) - assert.Equal(t, float64(4), hello["revpos"]) + + // revpos should match the generation of the current revision, since it's new to SGW with that revision. + // The actual revTreeID will differ when running this test as HLV client (multiple updates to HLV on client + // don't result in multiple revTree revisions) + expectedRevPos, _ := db.ParseRevID(ctx, doc.CurrentRev) + assert.Equal(t, float64(expectedRevPos), hello["revpos"]) assert.True(t, hello["stub"].(bool)) // Check the number of sendProveAttachment/sendGetAttachment calls. @@ -532,7 +550,6 @@ func TestBlipAttachNameChange(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -582,7 +599,7 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires legacy attachment upgrade to HLV (CBG-3806) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -602,10 +619,10 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { CreateDocWithLegacyAttachment(t, client1.rt, docID, rawDoc, attKey, attBody) // Get the document and grab the revID. - docVersion, _ := client1.rt.GetDoc(docID) + docVersion := client1.GetDocVersion(docID) // Store the document and attachment on the test client - err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.RevTreeID, rawDoc) + err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.GetRev(client1.UseHLV()), rawDoc) require.NoError(t, err) btcRunner.AttachmentsLock(client1.id).Lock() @@ -640,7 +657,7 @@ func TestBlipLegacyAttachDocUpdate(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires legacy attachment upgrade to HLV (CBG-3806) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) diff --git a/rest/blip_api_crud_test.go b/rest/blip_api_crud_test.go index b797c685a5..d768a3284d 100644 --- a/rest/blip_api_crud_test.go +++ b/rest/blip_api_crud_test.go @@ -2580,7 +2580,6 @@ func TestBlipInternalPropertiesHandling(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires HLV revpos handling (CBG-3797) for _attachments subtest btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { // Setup diff --git a/rest/blip_client_test.go b/rest/blip_client_test.go index 3479db1d84..e9b0661591 100644 --- a/rest/blip_client_test.go +++ b/rest/blip_client_test.go @@ -848,7 +848,9 @@ func (btr *BlipTesterReplicator) sendMsg(msg *blip.Message) (err error) { // PushRev creates a revision on the client, and immediately sends a changes request for it. // The rev ID is always: "N-abc", where N is rev generation for predictability. func (btc *BlipTesterCollectionClient) PushRev(docID string, parentVersion DocVersion, body []byte) (DocVersion, error) { - revID, err := btc.PushRevWithHistory(docID, parentVersion.RevTreeID, body, 1, 0) + + parentRev := parentVersion.GetRev(btc.UseHLV()) + revID, err := btc.PushRevWithHistory(docID, parentRev, body, 1, 0) if err != nil { return DocVersion{}, err } @@ -867,13 +869,21 @@ func (btc *BlipTesterCollectionClient) requireRevID(expected DocVersion, revID s // GetDocVersion fetches revid and cv directly from the bucket. Used to support REST-based verification in btc tests // even while REST only supports revTreeId -func (btc *BlipTesterCollectionClient) GetDocVersion(docID string) DocVersion { +// TODO: This doesn't support multi-collection testing, btc.GetDocVersion uses +// +// GetSingleTestDatabaseCollection() +func (btcc *BlipTesterCollectionClient) GetDocVersion(docID string) DocVersion { + return btcc.parent.GetDocVersion(docID) +} - collection := btc.parent.rt.GetSingleTestDatabaseCollection() - ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB), btc.parent.rt.GetDatabase().Name, nil) +// GetDocVersion fetches revid and cv directly from the bucket. Used to support REST-based verification in btc tests +// even while REST only supports revTreeId +func (btc *BlipTesterClient) GetDocVersion(docID string) DocVersion { + collection := btc.rt.GetSingleTestDatabaseCollection() + ctx := base.DatabaseLogCtx(base.TestCtx(btc.rt.TB), btc.rt.GetDatabase().Name, nil) doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalSync) - require.NoError(btc.parent.rt.TB, err) - if !btc.UseHLV() { + require.NoError(btc.rt.TB, err) + if !btc.UseHLV() || doc.HLV == nil { return DocVersion{RevTreeID: doc.CurrentRev} } return DocVersion{RevTreeID: doc.CurrentRev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} @@ -897,7 +907,7 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin startValue = parentVersion.Value revisionHistory = append(revisionHistory, parentRev) } - newVersion := db.DecodedVersion{SourceID: "abc", Value: startValue + uint64(revCount) + 1} + newVersion := db.DecodedVersion{SourceID: "abc", Value: startValue + uint64(revCount)} newRevID = newVersion.String() } else { @@ -1023,16 +1033,20 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin return newRevID, nil } -func (btc *BlipTesterCollectionClient) StoreRevOnClient(docID, revID string, body []byte) error { +func (btc *BlipTesterCollectionClient) StoreRevOnClient(docID, rev string, body []byte) error { ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB), btc.parent.rt.GetDatabase().Name, nil) - revGen, _ := db.ParseRevID(ctx, revID) + + revGen := 0 + if !btc.UseHLV() { + revGen, _ = db.ParseRevID(ctx, rev) + } newBody, err := btc.ProcessInlineAttachments(body, revGen) if err != nil { return err } btc.docsLock.Lock() defer btc.docsLock.Unlock() - btc.docs[docID] = btc.NewBlipTesterDoc(revID, newBody, nil) + btc.docs[docID] = btc.NewBlipTesterDoc(rev, newBody, nil) return nil } @@ -1376,3 +1390,28 @@ func (btc *BlipTesterCollectionClient) sendPushMsg(msg *blip.Message) error { btc.addCollectionProperty(msg) return btc.parent.pushReplication.sendMsg(msg) } + +// Wrappers for RT helpers that populate version information for use in HLV tests +// PutDoc will upsert the document with a given contents. +func (btc *BlipTesterClient) PutDoc(docID string, body string) DocVersion { + rt := btc.rt + version := rt.PutDoc(docID, body) + if btc.UseHLV() { + source, value := rt.GetSingleTestDatabaseCollection().GetDocumentCurrentVersion(rt.TB, docID) + version.CV = db.Version{ + SourceID: source, + Value: value, + } + } + return version +} + +// RequireRev checks the current rev for the specified docID on the backend the BTC is replicating +// with (NOT in the btc store) +func (btc *BlipTesterClient) RequireRev(t *testing.T, expectedRev string, doc *db.Document) { + if btc.UseHLV() { + require.Equal(t, expectedRev, doc.HLV.GetCurrentVersionString()) + } else { + require.Equal(t, expectedRev, doc.CurrentRev) + } +} diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 85328d64d2..9a3f7e24a7 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -2368,6 +2368,17 @@ func (v DocVersion) Equal(o DocVersion) bool { return true } +func (v DocVersion) GetRev(useHLV bool) string { + if useHLV { + if v.CV.SourceID == "" { + return "" + } + return v.CV.String() + } else { + return v.RevTreeID + } +} + // Digest returns the digest for the current version func (v DocVersion) Digest() string { return strings.Split(v.RevTreeID, "-")[1]