From f6fb3413e96389ebeb60ed1702af31685b6eccaa Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 5 Dec 2024 11:27:13 -0500 Subject: [PATCH] CBG-4366 enable resurrection tests (#7229) --- rest/utilities_testing_resttester.go | 5 + topologytest/couchbase_lite_mock_peer_test.go | 5 + topologytest/couchbase_server_peer_test.go | 4 + topologytest/hlv_test.go | 189 ++++++++++-------- topologytest/peer_test.go | 31 ++- topologytest/sync_gateway_peer_test.go | 7 +- topologytest/topologies_test.go | 2 - 7 files changed, 144 insertions(+), 99 deletions(-) diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 07dc7e56e5..4fd150d3f3 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -36,6 +36,11 @@ func (rt *RestTester) Run(name string, test func(*testing.T)) { }) } +func (rt *RestTester) UpdateTB(t *testing.T) { + var tb testing.TB = t + rt.testingTB.Store(&tb) +} + // GetDocBody returns the doc body for the given docID. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocBody(docID string) db.Body { rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID, "") diff --git a/topologytest/couchbase_lite_mock_peer_test.go b/topologytest/couchbase_lite_mock_peer_test.go index bf0dd6cce3..4ab1b333aa 100644 --- a/topologytest/couchbase_lite_mock_peer_test.go +++ b/topologytest/couchbase_lite_mock_peer_test.go @@ -170,6 +170,11 @@ func (p *CouchbaseLiteMockPeer) TB() testing.TB { return p.t } +// UpdateTB updates the testing.TB for the peer. +func (p *CouchbaseLiteMockPeer) UpdateTB(t *testing.T) { + p.t = t +} + // GetBackingBucket returns the backing bucket for the peer. This is always nil. func (p *CouchbaseLiteMockPeer) GetBackingBucket() base.Bucket { return nil diff --git a/topologytest/couchbase_server_peer_test.go b/topologytest/couchbase_server_peer_test.go index 67b4d6d907..3049635f17 100644 --- a/topologytest/couchbase_server_peer_test.go +++ b/topologytest/couchbase_server_peer_test.go @@ -276,6 +276,10 @@ func (p *CouchbaseServerPeer) TB() testing.TB { return p.tb } +func (p *CouchbaseServerPeer) UpdateTB(tb *testing.T) { + p.tb = tb +} + // getDocVersion returns a DocVersion from a cas and xattrs with _vv (hlv) and _sync (RevTreeID). func getDocVersion(docID string, peer Peer, cas uint64, xattrs map[string][]byte) DocMetadata { docVersion := DocMetadata{ diff --git a/topologytest/hlv_test.go b/topologytest/hlv_test.go index 7a34206b41..9eb75f9279 100644 --- a/topologytest/hlv_test.go +++ b/topologytest/hlv_test.go @@ -56,17 +56,6 @@ func (t singleActorTest) collectionName() base.ScopeAndCollectionName { return getSingleDsName() } -// getSingleActorTestCase returns a list of test cases in the matrix for all topologies * active peers. -func getSingleActorTestCase() []singleActorTest { - var tests []singleActorTest - for _, tc := range append(simpleTopologies, Topologies...) { - for _, activePeerID := range tc.PeerNames() { - tests = append(tests, singleActorTest{topology: tc, activePeerID: activePeerID}) - } - } - return tests -} - // multiActorTest represents a test case for a single actor in a given topology. type multiActorTest struct { topology Topology @@ -118,14 +107,19 @@ func startPeerReplications(peerReplications []PeerReplication) { func TestHLVCreateDocumentSingleActor(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyImport, base.KeyVV) - for _, tc := range getSingleActorTestCase() { - t.Run(tc.description(), func(t *testing.T) { - peers, _ := setupTests(t, tc.topology, tc.activePeerID) - - docID := getDocID(t) - docBody := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s"}`, tc.activePeerID, tc.description())) - docVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, docBody) - waitForVersionAndBody(t, tc, peers, docID, docVersion) + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + peers, _ := setupTests(t, topology) + for _, activePeerID := range topology.PeerNames() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + tc := singleActorTest{topology: topology, activePeerID: activePeerID} + docID := getDocID(t) + docBody := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s"}`, activePeerID, tc.description())) + docVersion := peers[activePeerID].CreateDocument(getSingleDsName(), docID, docBody) + waitForVersionAndBody(t, tc, peers, docID, docVersion) + }) + } }) } } @@ -135,7 +129,7 @@ func TestHLVCreateDocumentMultiActor(t *testing.T) { for _, tc := range getMultiActorTestCases() { t.Run(tc.description(), func(t *testing.T) { - peers, _ := setupTests(t, tc.topology, "") + peers, _ := setupTests(t, tc.topology) var docVersionList []BodyAndVersion @@ -173,7 +167,7 @@ func TestHLVCreateDocumentMultiActorConflict(t *testing.T) { t.Skip("We need to be able to wait for a specific version to arrive over pull replication, CBG-4257") } t.Run(tc.description(), func(t *testing.T) { - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) @@ -196,7 +190,7 @@ func TestHLVUpdateDocumentMultiActor(t *testing.T) { if strings.Contains(tc.description(), "CBL") { t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257") } - peers, _ := setupTests(t, tc.topology, "") + peers, _ := setupTests(t, tc.topology) // grab sorted peer list and create a list to store expected version, // doc body and the peer the write came from @@ -228,25 +222,32 @@ func TestHLVUpdateDocumentMultiActor(t *testing.T) { func TestHLVUpdateDocumentSingleActor(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyImport, base.KeyVV) - for _, tc := range getSingleActorTestCase() { - t.Run(tc.description(), func(t *testing.T) { - if strings.HasPrefix(tc.activePeerID, "cbl") { - t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257") - } - peers, _ := setupTests(t, tc.topology, tc.activePeerID) - docID := getDocID(t) - body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) - createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + peers, _ := setupTests(t, topology) + for _, activePeerID := range topology.PeerNames() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + tc := singleActorTest{topology: topology, activePeerID: activePeerID} + if strings.HasPrefix(tc.activePeerID, "cbl") { + t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257") + } + + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) + createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) - waitForVersionAndBody(t, tc, peers, docID, createVersion) + waitForVersionAndBody(t, tc, peers, docID, createVersion) - body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 2}`, tc.activePeerID, tc.description())) - updateVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2) - t.Logf("createVersion: %+v, updateVersion: %+v", createVersion.docMeta, updateVersion.docMeta) - t.Logf("waiting for document version 2 on all peers") + body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 2}`, tc.activePeerID, tc.description())) + updateVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2) + t.Logf("createVersion: %+v, updateVersion: %+v", createVersion.docMeta, updateVersion.docMeta) + t.Logf("waiting for document version 2 on all peers") - waitForVersionAndBody(t, tc, peers, docID, updateVersion) + waitForVersionAndBody(t, tc, peers, docID, updateVersion) + }) + } }) } } @@ -272,7 +273,7 @@ func TestHLVUpdateDocumentMultiActorConflict(t *testing.T) { t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") } t.Run(tc.description(), func(t *testing.T) { - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) docID := getDocID(t) @@ -294,23 +295,31 @@ func TestHLVUpdateDocumentMultiActorConflict(t *testing.T) { func TestHLVDeleteDocumentSingleActor(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport, base.KeyVV) - for _, tc := range getSingleActorTestCase() { - t.Run(tc.description(), func(t *testing.T) { - if strings.HasPrefix(tc.activePeerID, "cbl") { - t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") - } - peers, _ := setupTests(t, tc.topology, tc.activePeerID) - docID := getDocID(t) - body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) - createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + peers, _ := setupTests(t, topology) + for _, activePeerID := range topology.PeerNames() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + tc := singleActorTest{topology: topology, activePeerID: activePeerID} - waitForVersionAndBody(t, tc, peers, docID, createVersion) + if strings.HasPrefix(tc.activePeerID, "cbl") { + t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") + } + + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) + createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) - deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID) - t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion.docMeta, deleteVersion) - t.Logf("waiting for document deletion on all peers") - waitForDeletion(t, tc, peers, docID, tc.activePeerID) + waitForVersionAndBody(t, tc, peers, docID, createVersion) + + deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID) + t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion.docMeta, deleteVersion) + t.Logf("waiting for document deletion on all peers") + waitForDeletion(t, tc, peers, docID, tc.activePeerID) + }) + } }) } } @@ -323,7 +332,7 @@ func TestHLVDeleteDocumentMultiActor(t *testing.T) { if strings.Contains(tc.description(), "CBL") { t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") } - peers, _ := setupTests(t, tc.topology, "") + peers, _ := setupTests(t, tc.topology) for peerName := range peers { docID := getDocID(t) + "_" + peerName @@ -368,7 +377,7 @@ func TestHLVDeleteDocumentMultiActorConflict(t *testing.T) { t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") } t.Run(tc.description(), func(t *testing.T) { - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) docID := getDocID(t) @@ -408,7 +417,7 @@ func TestHLVUpdateDeleteDocumentMultiActorConflict(t *testing.T) { } t.Run(tc.description(), func(t *testing.T) { peerList := tc.PeerNames() - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) docID := getDocID(t) @@ -453,7 +462,7 @@ func TestHLVDeleteUpdateDocumentMultiActorConflict(t *testing.T) { } t.Run(tc.description(), func(t *testing.T) { peerList := tc.PeerNames() - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) docID := getDocID(t) @@ -481,38 +490,43 @@ func TestHLVDeleteUpdateDocumentMultiActorConflict(t *testing.T) { func TestHLVResurrectDocumentSingleActor(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport, base.KeyVV) - for _, tc := range getSingleActorTestCase() { - t.Run(tc.description(), func(t *testing.T) { - if strings.HasPrefix(tc.activePeerID, "cbl") { - t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") - } - t.Skip("Skipping resurection tests CBG-4366") + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + peers, _ := setupTests(t, topology) + for _, activePeerID := range topology.PeerNames() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + tc := singleActorTest{topology: topology, activePeerID: activePeerID} + + if strings.HasPrefix(tc.activePeerID, "cbl") { + t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") + } - peers, _ := setupTests(t, tc.topology, tc.activePeerID) + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) + createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) + waitForVersionAndBody(t, tc, peers, docID, createVersion) - docID := getDocID(t) - body1 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": 1}`, tc.activePeerID, tc.description())) - createVersion := peers[tc.activePeerID].CreateDocument(tc.collectionName(), docID, body1) - waitForVersionAndBody(t, tc, peers, docID, createVersion) - - deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID) - t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion, deleteVersion) - t.Logf("waiting for document deletion on all peers") - waitForDeletion(t, tc, peers, docID, tc.activePeerID) - - body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": "resurrection"}`, tc.activePeerID, tc.description())) - resurrectVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2) - t.Logf("createVersion: %+v, deleteVersion: %+v, resurrectVersion: %+v", createVersion.docMeta, deleteVersion, resurrectVersion.docMeta) - t.Logf("waiting for document resurrection on all peers") - - // Couchbase Lite peers do not know how to push a deletion yet, so we need to filter them out CBG-4257 - nonCBLPeers := make(map[string]Peer) - for peerName, peer := range peers { - if !strings.HasPrefix(peerName, "cbl") { - nonCBLPeers[peerName] = peer - } + deleteVersion := peers[tc.activePeerID].DeleteDocument(tc.collectionName(), docID) + t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion, deleteVersion) + t.Logf("waiting for document deletion on all peers") + waitForDeletion(t, tc, peers, docID, tc.activePeerID) + + body2 := []byte(fmt.Sprintf(`{"peer": "%s", "topology": "%s", "write": "resurrection"}`, tc.activePeerID, tc.description())) + resurrectVersion := peers[tc.activePeerID].WriteDocument(tc.collectionName(), docID, body2) + t.Logf("createVersion: %+v, deleteVersion: %+v, resurrectVersion: %+v", createVersion.docMeta, deleteVersion, resurrectVersion.docMeta) + t.Logf("waiting for document resurrection on all peers") + + // Couchbase Lite peers do not know how to push a deletion yet, so we need to filter them out CBG-4257 + nonCBLPeers := make(map[string]Peer) + for peerName, peer := range peers { + if !strings.HasPrefix(peerName, "cbl") { + nonCBLPeers[peerName] = peer + } + } + waitForVersionAndBody(t, tc, peers, docID, resurrectVersion) + }) } - waitForVersionAndBody(t, tc, peers, docID, resurrectVersion) }) } } @@ -524,9 +538,8 @@ func TestHLVResurrectDocumentMultiActor(t *testing.T) { if strings.Contains(tc.description(), "CBL") { t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") } - t.Skip("skipped resurrection test, intermittent failures CBG-4372") - peers, _ := setupTests(t, tc.topology, "") + peers, _ := setupTests(t, tc.topology) var docVersionList []BodyAndVersion for _, peerName := range tc.PeerNames() { @@ -578,7 +591,7 @@ func TestHLVResurrectDocumentMultiActorConflict(t *testing.T) { t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") } t.Run(tc.description(), func(t *testing.T) { - peers, replications := setupTests(t, tc.topology, "") + peers, replications := setupTests(t, tc.topology) stopPeerReplications(replications) docID := getDocID(t) diff --git a/topologytest/peer_test.go b/topologytest/peer_test.go index 7c377866ec..e006a593b6 100644 --- a/topologytest/peer_test.go +++ b/topologytest/peer_test.go @@ -22,10 +22,17 @@ import ( "github.com/stretchr/testify/require" ) -const ( - totalWaitTime = 10 * time.Second - pollInterval = 50 * time.Millisecond -) +// totalWaitTime is the time to wait for a document on a peer. This time is low for rosmar and high for Couchbase Server. +var totalWaitTime = 3 * time.Second + +// pollInterval is the time to poll to see if a document is updated on a peer +var pollInterval = 50 * time.Millisecond + +func init() { + if !base.UnitTestUrlIsWalrus() { + totalWaitTime = 40 * time.Second + } +} // Peer represents a peer in an Mobile workflow. The types of Peers are Couchbase Server, Sync Gateway, or Couchbase Lite. type Peer interface { @@ -70,6 +77,9 @@ type internalPeer interface { // TB returns the testing.TB for the peer. TB() testing.TB + // UpdateTB updates the testing.TB for the peer. + UpdateTB(*testing.T) + // Context returns the context for the peer. Context() context.Context } @@ -239,14 +249,19 @@ func createPeers(t *testing.T, peersOptions map[string]PeerOptions) map[string]P return peers } +func updatePeersT(t *testing.T, peers map[string]Peer) { + for _, peer := range peers { + oldTB := peer.TB().(*testing.T) + t.Cleanup(func() { peer.UpdateTB(oldTB) }) + peer.UpdateTB(t) + } +} + // setupTests returns a map of peers and a list of replications. The peers will be closed and the buckets will be destroyed by t.Cleanup. -func setupTests(t *testing.T, topology Topology, activePeerID string) (map[string]Peer, []PeerReplication) { +func setupTests(t *testing.T, topology Topology) (map[string]Peer, []PeerReplication) { peers := createPeers(t, topology.peers) replications := createPeerReplications(t, peers, topology.replications) - if topology.skipIf != nil { - topology.skipIf(t, activePeerID, peers) - } for _, replication := range replications { // temporarily start the replication before writing the document, limitation of CouchbaseLiteMockPeer as active peer since WriteDocument is calls PushRev replication.Start() diff --git a/topologytest/sync_gateway_peer_test.go b/topologytest/sync_gateway_peer_test.go index 5ff62fbec0..2486e5df8e 100644 --- a/topologytest/sync_gateway_peer_test.go +++ b/topologytest/sync_gateway_peer_test.go @@ -150,7 +150,7 @@ func (p *SyncGatewayPeer) WaitForDeletion(dsName sgbucket.DataStoreName, docID s require.EventuallyWithT(p.TB(), func(c *assert.CollectT) { doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) if err == nil { - assert.True(c, doc.IsDeleted(), "expected %s on %s to be deleted", doc, p) + assert.True(c, doc.IsDeleted(), "expected %+v on %s to be deleted", doc, p) return } assert.True(c, base.IsDocNotFoundError(err), "expected docID %s on %s to be deleted, found err=%v", docID, p, err) @@ -200,6 +200,11 @@ func (p *SyncGatewayPeer) TB() testing.TB { return p.rt.TB() } +// UpdateTB updates the testing.TB for the peer. +func (p *SyncGatewayPeer) UpdateTB(t *testing.T) { + p.rt.UpdateTB(t) +} + // GetBackingBucket returns the backing bucket for the peer. func (p *SyncGatewayPeer) GetBackingBucket() base.Bucket { return p.rt.Bucket() diff --git a/topologytest/topologies_test.go b/topologytest/topologies_test.go index 9a9055c996..408b9dcb5f 100644 --- a/topologytest/topologies_test.go +++ b/topologytest/topologies_test.go @@ -10,7 +10,6 @@ package topologytest import ( "slices" - "testing" "golang.org/x/exp/maps" ) @@ -20,7 +19,6 @@ type Topology struct { description string peers map[string]PeerOptions replications []PeerReplicationDefinition - skipIf func(t *testing.T, activePeerID string, peers map[string]Peer) // allow temporary skips while the code is being ironed out } // PeerNames returns a sorted list of peers.