CBG-3788 Support HLV operations in BlipTesterClient (#6689)

* CBG-3788 Support HLV operations in BlipTesterClient

Switches the BlipTesterCollectionClient to maintain client HLV and (linear) revtree per document.  Switches the docs struct to a map of a new BlipTesterDoc struct, instead of a map of revs per document.

BlipTesterDoc still maintains a history of all rev messages received (revMessageHistory) to support test evaluation of received messages, but also defines a linear revTreeId history or an HLV (depending on protocol enabled for the test).

Includes a refactor of revID to revTreeID in RevAndVersion, as a step toward standardizing ‘revID’ as the generic property used during replication (which can be currentRev or cv), and revTreeID as a traditional revtree revision ID.

* Fixes based on PR review

db/blip_sync_context.go

@@ -623,7 +623,7 @@ func (bsc *BlipSyncContext) sendRevision(sender *blip.Sender, docID, rev string,
 		docRev, err = handleChangesResponseCollection.GetRev(bsc.loggingCtx, docID, rev, true, nil)
 	} else {
 		// extract CV string rev representation
-		version, vrsErr := CreateVersionFromString(rev)
+		version, vrsErr := ParseVersion(rev)
 		if vrsErr != nil {
 			return vrsErr
 		}

db/crud.go

@@ -2233,7 +2233,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do
 			Expiry:           doc.Expiry,
 			Deleted:          doc.History[newRevID].Deleted,
 			_shallowCopyBody: storedDoc.Body(ctx),
-			hlvHistory:       doc.HLV.toHistoryForHLV(),
+			hlvHistory:       doc.HLV.ToHistoryForHLV(),
 			CV:               &Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version},
 		}
 

db/hybrid_logical_vector.go

@@ -70,7 +70,7 @@ func CreateVersion(source, version string) Version {
 	}
 }
 
-func CreateVersionFromString(versionString string) (version Version, err error) {
+func ParseVersion(versionString string) (version Version, err error) {
 	timestampString, sourceBase64, found := strings.Cut(versionString, "@")
 	if !found {
 		return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString)
@@ -80,6 +80,16 @@ func CreateVersionFromString(versionString string) (version Version, err error)
 	return version, nil
 }
 
+func ParseDecodedVersion(versionString string) (version DecodedVersion, err error) {
+	timestampString, sourceBase64, found := strings.Cut(versionString, "@")
+	if !found {
+		return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString)
+	}
+	version.SourceID = sourceBase64
+	version.Value = base.HexCasToUint64(timestampString)
+	return version, nil
+}
+
 // String returns a Couchbase Lite-compatible string representation of the version.
 func (v DecodedVersion) String() string {
 	timestamp := string(base.Uint64CASToLittleEndianHex(v.Value))
@@ -178,7 +188,7 @@ func (hlv *HybridLogicalVector) AddVersion(newVersion Version) error {
 		if currPVVersionCAS < hlvVersionCAS {
 			hlv.PreviousVersions[hlv.SourceID] = hlv.Version
 		} else {
-			return fmt.Errorf("local hlv has current source in previous versiosn with version greater than current version. Current CAS: %s, PV CAS %s", hlv.Version, currPVVersion)
+			return fmt.Errorf("local hlv has current source in previous version with version greater than current version. Current CAS: %s, PV CAS %s", hlv.Version, currPVVersion)
 		}
 	} else {
 		// source doesn't exist in PV so add
@@ -375,8 +385,16 @@ func (hlv *HybridLogicalVector) setPreviousVersion(source string, version string
 	hlv.PreviousVersions[source] = version
 }
 
+func (hlv *HybridLogicalVector) IsVersionKnown(otherVersion Version) bool {
+	value, found := hlv.GetVersion(otherVersion.SourceID)
+	if !found {
+		return false
+	}
+	return value >= base.HexCasToUint64(otherVersion.Value)
+}
+
 // toHistoryForHLV formats blip History property for V4 replication and above
-func (hlv *HybridLogicalVector) toHistoryForHLV() string {
+func (hlv *HybridLogicalVector) ToHistoryForHLV() string {
 	// take pv and mv from hlv if defined and add to history
 	var s strings.Builder
 	// Merge versions must be defined first if they exist

db/hybrid_logical_vector_test.go

@@ -332,7 +332,7 @@ func TestHLVMapToCBLString(t *testing.T) {
 	for _, test := range testCases {
 		t.Run(test.name, func(t *testing.T) {
 			hlv := createHLVForTest(t, test.inputHLV)
-			historyStr := hlv.toHistoryForHLV()
+			historyStr := hlv.ToHistoryForHLV()
 
 			if test.both {
 				initial := strings.Split(historyStr, ";")

db/revision_cache_bypass.go

@@ -52,7 +52,7 @@ func (rc *BypassRevisionCache) GetWithRev(ctx context.Context, docID, revID stri
 	}
 	if hlv != nil {
 		docRev.CV = hlv.ExtractCurrentVersionFromHLV()
-		docRev.hlvHistory = hlv.toHistoryForHLV()
+		docRev.hlvHistory = hlv.ToHistoryForHLV()
 	}
 
 	rc.bypassStat.Add(1)
@@ -80,7 +80,7 @@ func (rc *BypassRevisionCache) GetWithCV(ctx context.Context, docID string, cv *
 	}
 	if hlv != nil {
 		docRev.CV = hlv.ExtractCurrentVersionFromHLV()
-		docRev.hlvHistory = hlv.toHistoryForHLV()
+		docRev.hlvHistory = hlv.ToHistoryForHLV()
 	}
 
 	rc.bypassStat.Add(1)
@@ -111,7 +111,7 @@ func (rc *BypassRevisionCache) GetActive(ctx context.Context, docID string, incl
 	}
 	if hlv != nil {
 		docRev.CV = hlv.ExtractCurrentVersionFromHLV()
-		docRev.hlvHistory = hlv.toHistoryForHLV()
+		docRev.hlvHistory = hlv.ToHistoryForHLV()
 	}
 
 	rc.bypassStat.Add(1)

db/revision_cache_lru.go

@@ -544,15 +544,15 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache
 			// based off the current value load we need to populate the revid key with what has been fetched from the bucket (for use of populating the opposite lookup map)
 			value.revID = revid
 			if hlv != nil {
-				value.hlvHistory = hlv.toHistoryForHLV()
+				value.hlvHistory = hlv.ToHistoryForHLV()
 			}
 		} else {
 			revKey := IDAndRev{DocID: value.id, RevID: value.revID}
 			value.bodyBytes, value.body, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, hlv, value.err = revCacheLoader(ctx, backingStore, revKey, includeBody)
 			// based off the revision load we need to populate the hlv key with what has been fetched from the bucket (for use of populating the opposite lookup map)
 			if hlv != nil {
 				value.cv = *hlv.ExtractCurrentVersionFromHLV()
-				value.hlvHistory = hlv.toHistoryForHLV()
+				value.hlvHistory = hlv.ToHistoryForHLV()
 			}
 		}
 	}
@@ -655,12 +655,12 @@ func (value *revCacheValue) loadForDoc(ctx context.Context, backingStore Revisio
 		if value.revID == "" {
 			value.bodyBytes, value.body, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, revid, hlv, value.err = revCacheLoaderForDocumentCV(ctx, backingStore, doc, value.cv)
 			value.revID = revid
-			value.hlvHistory = hlv.toHistoryForHLV()
+			value.hlvHistory = hlv.ToHistoryForHLV()
 		} else {
 			value.bodyBytes, value.body, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, hlv, value.err = revCacheLoaderForDocument(ctx, backingStore, doc, value.revID)
 			if hlv != nil {
 				value.cv = *hlv.ExtractCurrentVersionFromHLV()
-				value.hlvHistory = hlv.toHistoryForHLV()
+				value.hlvHistory = hlv.ToHistoryForHLV()
 			}
 		}
 	}

rest/access_test.go

@@ -414,27 +414,27 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assertRespStatus(resp, http.StatusForbidden)
 
 			// User has no permissions to access rev
-			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password")
+			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password")
 			assertRespStatus(resp, http.StatusOK)
 
 			// Guest has no permissions to access rev
-			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "")
+			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "")
 			assertRespStatus(resp, http.StatusOK)
 
 			// Attachments should be forbidden as well
 			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach", "", nil, "NoPerms", "password")
 			assertRespStatus(resp, http.StatusForbidden)
 
 			// Attachment revs should be forbidden as well
-			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "NoPerms", "password")
+			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "NoPerms", "password")
 			assertRespStatus(resp, http.StatusNotFound)
 
 			// Attachments should be forbidden for guests as well
 			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach", "", nil, "", "")
 			assertRespStatus(resp, http.StatusForbidden)
 
 			// Attachment revs should be forbidden for guests as well
-			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "", "")
+			resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "", "")
 			assertRespStatus(resp, http.StatusNotFound)
 
 			// Document does not exist should cause 403
@@ -451,7 +451,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assertRespStatus(resp, http.StatusConflict)
 
 			// PUT with rev
-			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "NoPerms", "password")
+			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "NoPerms", "password")
 			assertRespStatus(resp, http.StatusForbidden)
 
 			// PUT with incorrect rev
@@ -463,7 +463,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assertRespStatus(resp, http.StatusConflict)
 
 			// PUT with rev as Guest
-			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "", "")
+			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "", "")
 			assertRespStatus(resp, http.StatusForbidden)
 
 			// PUT with incorrect rev as Guest
@@ -495,7 +495,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assert.NotContains(t, user.GetChannels(s, c).ToArray(), "chan2")
 
 			// Successful PUT which will grant access grants
-			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{"channels": "chan"}`, nil, "Perms", "password")
+			resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{"channels": "chan"}`, nil, "Perms", "password")
 			AssertStatus(t, resp, http.StatusCreated)
 
 			// Make sure channel access grant was successful
@@ -512,7 +512,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assertRespStatus(resp, http.StatusConflict)
 
 			// Attempt to delete document rev with no permissions
-			resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password")
+			resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password")
 			assertRespStatus(resp, http.StatusConflict)
 
 			// Attempt to delete document with wrong rev
@@ -528,7 +528,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 			assertRespStatus(resp, http.StatusConflict)
 
 			// Attempt to delete document rev with no write perms as guest
-			resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "")
+			resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "")
 			assertRespStatus(resp, http.StatusConflict)
 
 			// Attempt to delete document with wrong rev as guest
@@ -1184,7 +1184,7 @@ func TestRoleChannelGrantInheritance(t *testing.T) {
 	RequireStatus(t, response, 200)
 
 	// Revoke access to chan2 (dynamic)
-	response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1")
+	response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevTreeID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1")
 	RequireStatus(t, response, 201)
 
 	// Verify user cannot access doc in revoked channel, but can successfully access remaining documents

rest/adminapitest/admin_api_test.go

@@ -2144,7 +2144,7 @@ func TestRawTombstone(t *testing.T) {
 	resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``)
 	assert.Equal(t, "application/json", resp.Header().Get("Content-Type"))
 	assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`)
-	assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevID+`"`)
+	assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevTreeID+`"`)
 	assert.Contains(t, string(resp.BodyBytes()), `"foo":"bar"`)
 	assert.NotContains(t, string(resp.BodyBytes()), `"_deleted":true`)
 
@@ -2154,7 +2154,7 @@ func TestRawTombstone(t *testing.T) {
 	resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``)
 	assert.Equal(t, "application/json", resp.Header().Get("Content-Type"))
 	assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`)
-	assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevID+`"`)
+	assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevTreeID+`"`)
 	assert.NotContains(t, string(resp.BodyBytes()), `"foo":"bar"`)
 	assert.Contains(t, string(resp.BodyBytes()), `"_deleted":true`)
 }
@@ -3881,9 +3881,9 @@ func TestPutIDRevMatchBody(t *testing.T) {
 			docRev := test.rev
 			docBody := test.docBody
 			if test.docID == "" {
-				docID = "doc"                                                 // Used for the rev tests to branch off of
-				docBody = strings.ReplaceAll(docBody, "[REV]", version.RevID) // FIX for HLV?
-				docRev = strings.ReplaceAll(docRev, "[REV]", version.RevID)
+				docID = "doc"                                                     // Used for the rev tests to branch off of
+				docBody = strings.ReplaceAll(docBody, "[REV]", version.RevTreeID) // FIX for HLV?
+				docRev = strings.ReplaceAll(docRev, "[REV]", version.RevTreeID)
 			}
 
 			resp := rt.SendAdminRequest("PUT", fmt.Sprintf("/{{.keyspace}}/%s?rev=%s", docID, docRev), docBody)

rest/api_test.go

@@ -211,9 +211,9 @@ func TestDocLifecycle(t *testing.T) {
 	defer rt.Close()
 
 	version := rt.CreateTestDoc("doc")
-	assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevID)
+	assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevTreeID)
 
-	response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevID, "")
+	response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevTreeID, "")
 	RequireStatus(t, response, 200)
 }
 

rest/api_test_helpers.go

@@ -50,13 +50,13 @@ func (rt *RestTester) PutNewEditsFalse(docID string, newVersion DocVersion, pare
 	require.NoError(rt.TB, marshalErr)
 
 	requestBody := body.ShallowCopy()
-	newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB), newVersion.RevID)
+	newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB), newVersion.RevTreeID)
 
 	revisions := make(map[string]interface{})
 	revisions["start"] = newRevGeneration
 	ids := []string{newRevDigest}
-	if parentVersion.RevID != "" {
-		_, parentDigest := db.ParseRevID(base.TestCtx(rt.TB), parentVersion.RevID)
+	if parentVersion.RevTreeID != "" {
+		_, parentDigest := db.ParseRevID(base.TestCtx(rt.TB), parentVersion.RevTreeID)
 		ids = append(ids, parentDigest)
 	}
 	revisions["ids"] = ids