Skip to content

Commit

Permalink
CBG-4366 enable resurrection tests (#7229)
Browse files Browse the repository at this point in the history
  • Loading branch information
@torcolvin @bbrks
torcolvin authored and bbrks committed Dec 5, 2024
1 parent 4fc9df0 commit f6fb341
Show file tree
Hide file tree
Showing 7 changed files with 144 additions and 99 deletions.
5 changes: 5 additions & 0 deletions rest/utilities_testing_resttester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ func (rt *RestTester) Run(name string, test func(*testing.T)) {
})
}

func (rt *RestTester) UpdateTB(t *testing.T) {
var tb testing.TB = t
rt.testingTB.Store(&tb)
}

// GetDocBody returns the doc body for the given docID. If the document is not found, t.Fail will be called.
func (rt *RestTester) GetDocBody(docID string) db.Body {
rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID, "")
Expand Down
5 changes: 5 additions & 0 deletions topologytest/couchbase_lite_mock_peer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,11 @@ func (p *CouchbaseLiteMockPeer) TB() testing.TB {
return p.t
}

// UpdateTB updates the testing.TB for the peer.
func (p *CouchbaseLiteMockPeer) UpdateTB(t *testing.T) {
p.t = t
}

// GetBackingBucket returns the backing bucket for the peer. This is always nil.
func (p *CouchbaseLiteMockPeer) GetBackingBucket() base.Bucket {
return nil
Expand Down
4 changes: 4 additions & 0 deletions topologytest/couchbase_server_peer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,10 @@ func (p *CouchbaseServerPeer) TB() testing.TB {
return p.tb
}

func (p *CouchbaseServerPeer) UpdateTB(tb *testing.T) {
p.tb = tb
}

// getDocVersion returns a DocVersion from a cas and xattrs with _vv (hlv) and _sync (RevTreeID).
func getDocVersion(docID string, peer Peer, cas uint64, xattrs map[string][]byte) DocMetadata {
docVersion := DocMetadata{
Expand Down
189 changes: 101 additions & 88 deletions topologytest/hlv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,6 @@ func (t singleActorTest) collectionName() base.ScopeAndCollectionName {
return getSingleDsName()
}

// getSingleActorTestCase returns a list of test cases in the matrix for all topologies * active peers.
func getSingleActorTestCase() []singleActorTest {
var tests []singleActorTest
for _, tc := range append(simpleTopologies, Topologies...) {
for _, activePeerID := range tc.PeerNames() {
tests = append(tests, singleActorTest{topology: tc, activePeerID: activePeerID})
}
}
return tests
}

// multiActorTest represents a test case for a single actor in a given topology.
type multiActorTest struct {
topology Topology
Expand Down Expand Up @@ -118,14 +107,19 @@ func startPeerReplications(peerReplications []PeerReplication) {
func TestHLVCreateDocumentSingleActor(t *testing.T) {

base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyImport, base.KeyVV)
for _, tc := range getSingleActorTestCase() {
t.Run(tc.description(), func(t *testing.T) {
peers, _ := setupTests(t, tc.topology, tc.activePeerID)

docID := getDocID(t)
docBody := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s"}`, tc.activePeerID, tc.description()))
docVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, docBody)
waitForVersionAndBody(t, tc, peers, docID, docVersion)
for _, topology := range append(simpleTopologies, Topologies...) {
t.Run(topology.description, func(t *testing.T) {
peers, _ := setupTests(t, topology)
for _, activePeerID := range topology.PeerNames() {
t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) {
updatePeersT(t, peers)
tc := singleActorTest{topology: topology, activePeerID: activePeerID}
docID := getDocID(t)
docBody := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s"}`, activePeerID, tc.description()))
docVersion := peers[activePeerID].CreateDocument(getSingleDsName(), docID, docBody)
waitForVersionAndBody(t, tc, peers, docID, docVersion)
})
}
})
}
}
Expand All @@ -135,7 +129,7 @@ func TestHLVCreateDocumentMultiActor(t *testing.T) {

for _, tc := range getMultiActorTestCases() {
t.Run(tc.description(), func(t *testing.T) {
peers, _ := setupTests(t, tc.topology, "")
peers, _ := setupTests(t, tc.topology)

var docVersionList []BodyAndVersion

Expand Down Expand Up @@ -173,7 +167,7 @@ func TestHLVCreateDocumentMultiActorConflict(t *testing.T) {
t.Skip("We need to be able to wait for a specific version to arrive over pull replication, CBG-4257")
}
t.Run(tc.description(), func(t *testing.T) {
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)

stopPeerReplications(replications)

Expand All @@ -196,7 +190,7 @@ func TestHLVUpdateDocumentMultiActor(t *testing.T) {
if strings.Contains(tc.description(), "CBL") {
t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257")
}
peers, _ := setupTests(t, tc.topology, "")
peers, _ := setupTests(t, tc.topology)

// grab sorted peer list and create a list to store expected version,
// doc body and the peer the write came from
Expand Down Expand Up @@ -228,25 +222,32 @@ func TestHLVUpdateDocumentMultiActor(t *testing.T) {
func TestHLVUpdateDocumentSingleActor(t *testing.T) {

base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyImport, base.KeyVV)
for _, tc := range getSingleActorTestCase() {
t.Run(tc.description(), func(t *testing.T) {
if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257")
}
peers, _ := setupTests(t, tc.topology, tc.activePeerID)

docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)
for _, topology := range append(simpleTopologies, Topologies...) {
t.Run(topology.description, func(t *testing.T) {
peers, _ := setupTests(t, topology)
for _, activePeerID := range topology.PeerNames() {
t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) {
updatePeersT(t, peers)
tc := singleActorTest{topology: topology, activePeerID: activePeerID}
if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257")
}

docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)

waitForVersionAndBody(t, tc, peers, docID, createVersion)
waitForVersionAndBody(t, tc, peers, docID, createVersion)

body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 2}`, tc.activePeerID, tc.description()))
updateVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2)
t.Logf("createVersion: %+v, updateVersion: %+v", createVersion.docMeta, updateVersion.docMeta)
t.Logf("waiting for document version 2 on all peers")
body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 2}`, tc.activePeerID, tc.description()))
updateVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2)
t.Logf("createVersion: %+v, updateVersion: %+v", createVersion.docMeta, updateVersion.docMeta)
t.Logf("waiting for document version 2 on all peers")

waitForVersionAndBody(t, tc, peers, docID, updateVersion)
waitForVersionAndBody(t, tc, peers, docID, updateVersion)
})
}
})
}
}
Expand All @@ -272,7 +273,7 @@ func TestHLVUpdateDocumentMultiActorConflict(t *testing.T) {
t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257")
}
t.Run(tc.description(), func(t *testing.T) {
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)
stopPeerReplications(replications)

docID := getDocID(t)
Expand All @@ -294,23 +295,31 @@ func TestHLVUpdateDocumentMultiActorConflict(t *testing.T) {
func TestHLVDeleteDocumentSingleActor(t *testing.T) {

base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport, base.KeyVV)
for _, tc := range getSingleActorTestCase() {
t.Run(tc.description(), func(t *testing.T) {
if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}
peers, _ := setupTests(t, tc.topology, tc.activePeerID)

docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)
for _, topology := range append(simpleTopologies, Topologies...) {
t.Run(topology.description, func(t *testing.T) {
peers, _ := setupTests(t, topology)
for _, activePeerID := range topology.PeerNames() {
t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) {
updatePeersT(t, peers)
tc := singleActorTest{topology: topology, activePeerID: activePeerID}

waitForVersionAndBody(t, tc, peers, docID, createVersion)
if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}

docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)

deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID)
t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion.docMeta, deleteVersion)
t.Logf("waiting for document deletion on all peers")
waitForDeletion(t, tc, peers, docID, tc.activePeerID)
waitForVersionAndBody(t, tc, peers, docID, createVersion)

deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID)
t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion.docMeta, deleteVersion)
t.Logf("waiting for document deletion on all peers")
waitForDeletion(t, tc, peers, docID, tc.activePeerID)
})
}
})
}
}
Expand All @@ -323,7 +332,7 @@ func TestHLVDeleteDocumentMultiActor(t *testing.T) {
if strings.Contains(tc.description(), "CBL") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}
peers, _ := setupTests(t, tc.topology, "")
peers, _ := setupTests(t, tc.topology)

for peerName := range peers {
docID := getDocID(t) + "_" + peerName
Expand Down Expand Up @@ -368,7 +377,7 @@ func TestHLVDeleteDocumentMultiActorConflict(t *testing.T) {
t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257")
}
t.Run(tc.description(), func(t *testing.T) {
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)
stopPeerReplications(replications)

docID := getDocID(t)
Expand Down Expand Up @@ -408,7 +417,7 @@ func TestHLVUpdateDeleteDocumentMultiActorConflict(t *testing.T) {
}
t.Run(tc.description(), func(t *testing.T) {
peerList := tc.PeerNames()
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)
stopPeerReplications(replications)

docID := getDocID(t)
Expand Down Expand Up @@ -453,7 +462,7 @@ func TestHLVDeleteUpdateDocumentMultiActorConflict(t *testing.T) {
}
t.Run(tc.description(), func(t *testing.T) {
peerList := tc.PeerNames()
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)
stopPeerReplications(replications)

docID := getDocID(t)
Expand Down Expand Up @@ -481,38 +490,43 @@ func TestHLVDeleteUpdateDocumentMultiActorConflict(t *testing.T) {
func TestHLVResurrectDocumentSingleActor(t *testing.T) {

base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport, base.KeyVV)
for _, tc := range getSingleActorTestCase() {
t.Run(tc.description(), func(t *testing.T) {
if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}
t.Skip("Skipping resurection tests CBG-4366")
for _, topology := range append(simpleTopologies, Topologies...) {
t.Run(topology.description, func(t *testing.T) {
peers, _ := setupTests(t, topology)
for _, activePeerID := range topology.PeerNames() {
t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) {
updatePeersT(t, peers)
tc := singleActorTest{topology: topology, activePeerID: activePeerID}

if strings.HasPrefix(tc.activePeerID, "cbl") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}

peers, _ := setupTests(t, tc.topology, tc.activePeerID)
docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)
waitForVersionAndBody(t, tc, peers, docID, createVersion)

docID := getDocID(t)
body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description()))
createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1)
waitForVersionAndBody(t, tc, peers, docID, createVersion)

deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID)
t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion, deleteVersion)
t.Logf("waiting for document deletion on all peers")
waitForDeletion(t, tc, peers, docID, tc.activePeerID)

body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": "resurrection"}`, tc.activePeerID, tc.description()))
resurrectVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2)
t.Logf("createVersion: %+v, deleteVersion: %+v, resurrectVersion: %+v", createVersion.docMeta, deleteVersion, resurrectVersion.docMeta)
t.Logf("waiting for document resurrection on all peers")

// Couchbase Lite peers do not know how to push a deletion yet, so we need to filter them out CBG-4257
nonCBLPeers := make(map[string]Peer)
for peerName, peer := range peers {
if !strings.HasPrefix(peerName, "cbl") {
nonCBLPeers[peerName] = peer
}
deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID)
t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion, deleteVersion)
t.Logf("waiting for document deletion on all peers")
waitForDeletion(t, tc, peers, docID, tc.activePeerID)

body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": "resurrection"}`, tc.activePeerID, tc.description()))
resurrectVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2)
t.Logf("createVersion: %+v, deleteVersion: %+v, resurrectVersion: %+v", createVersion.docMeta, deleteVersion, resurrectVersion.docMeta)
t.Logf("waiting for document resurrection on all peers")

// Couchbase Lite peers do not know how to push a deletion yet, so we need to filter them out CBG-4257
nonCBLPeers := make(map[string]Peer)
for peerName, peer := range peers {
if !strings.HasPrefix(peerName, "cbl") {
nonCBLPeers[peerName] = peer
}
}
waitForVersionAndBody(t, tc, peers, docID, resurrectVersion)
})
}
waitForVersionAndBody(t, tc, peers, docID, resurrectVersion)
})
}
}
Expand All @@ -524,9 +538,8 @@ func TestHLVResurrectDocumentMultiActor(t *testing.T) {
if strings.Contains(tc.description(), "CBL") {
t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257")
}
t.Skip("skipped resurrection test, intermittent failures CBG-4372")

peers, _ := setupTests(t, tc.topology, "")
peers, _ := setupTests(t, tc.topology)

var docVersionList []BodyAndVersion
for _, peerName := range tc.PeerNames() {
Expand Down Expand Up @@ -578,7 +591,7 @@ func TestHLVResurrectDocumentMultiActorConflict(t *testing.T) {
t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257")
}
t.Run(tc.description(), func(t *testing.T) {
peers, replications := setupTests(t, tc.topology, "")
peers, replications := setupTests(t, tc.topology)
stopPeerReplications(replications)

docID := getDocID(t)
Expand Down
31 changes: 23 additions & 8 deletions topologytest/peer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,17 @@ import (
"github.com/stretchr/testify/require"
)

const (
totalWaitTime = 10 * time.Second
pollInterval = 50 * time.Millisecond
)
// totalWaitTime is the time to wait for a document on a peer. This time is low for rosmar and high for Couchbase Server.
var totalWaitTime = 3 * time.Second

// pollInterval is the time to poll to see if a document is updated on a peer
var pollInterval = 50 * time.Millisecond

func init() {
if !base.UnitTestUrlIsWalrus() {
totalWaitTime = 40 * time.Second
}
}

// Peer represents a peer in an Mobile workflow. The types of Peers are Couchbase Server, Sync Gateway, or Couchbase Lite.
type Peer interface {
Expand Down Expand Up @@ -70,6 +77,9 @@ type internalPeer interface {
// TB returns the testing.TB for the peer.
TB() testing.TB

// UpdateTB updates the testing.TB for the peer.
UpdateTB(*testing.T)

// Context returns the context for the peer.
Context() context.Context
}
Expand Down Expand Up @@ -239,14 +249,19 @@ func createPeers(t *testing.T, peersOptions map[string]PeerOptions) map[string]P
return peers
}

func updatePeersT(t *testing.T, peers map[string]Peer) {
for _, peer := range peers {
oldTB := peer.TB().(*testing.T)
t.Cleanup(func() { peer.UpdateTB(oldTB) })
peer.UpdateTB(t)
}
}

// setupTests returns a map of peers and a list of replications. The peers will be closed and the buckets will be destroyed by t.Cleanup.
func setupTests(t *testing.T, topology Topology, activePeerID string) (map[string]Peer, []PeerReplication) {
func setupTests(t *testing.T, topology Topology) (map[string]Peer, []PeerReplication) {
peers := createPeers(t, topology.peers)
replications := createPeerReplications(t, peers, topology.replications)

if topology.skipIf != nil {
topology.skipIf(t, activePeerID, peers)
}
for _, replication := range replications {
// temporarily start the replication before writing the document, limitation of CouchbaseLiteMockPeer as active peer since WriteDocument is calls PushRev
replication.Start()
Expand Down
Loading

0 comments on commit f6fb341

Please sign in to comment.