Skip to content

Commit

Permalink
Fix failed test
Browse files Browse the repository at this point in the history
  • Loading branch information
@hangc0276
hangc0276 committed Jan 22, 2024
1 parent 6a03165 commit e764366
Showing 1 changed file with 104 additions and 23 deletions.
127 changes: 104 additions & 23 deletions ...ar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import lombok.Cleanup;
import org.apache.bookkeeper.bookie.BookieImpl;
Expand Down Expand Up @@ -425,8 +426,16 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception {
// wait for 5 seconds before starting the recovery work when a bookie fails
urLedgerMgr.setLostBookieRecoveryDelay(5);

// shutdown a non auditor bookie; choosing non-auditor to avoid another election
String shutdownBookie = shutDownNonAuditorBookie();
AtomicReference<String> shutdownBookieRef = new AtomicReference<>();
CountDownLatch shutdownLatch = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie = shutDownNonAuditorBookie();
shutdownBookieRef.set(shutdownBookie);
shutdownLatch.countDown();
} catch (Exception ignore) {
}
}).start();

if (LOG.isDebugEnabled()) {
LOG.debug("Waiting for ledgers to be marked as under replicated");
Expand All @@ -442,9 +451,10 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception {
urLedgerList.contains(ledgerId));
Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
String data = urLedgerData.get(ledgerId);
assertTrue("Bookie " + shutdownBookie
shutdownLatch.await();
assertTrue("Bookie " + shutdownBookieRef.get()
+ "is not listed in the ledger as missing replica :" + data,
data.contains(shutdownBookie));
data.contains(shutdownBookieRef.get()));
}

/**
Expand Down Expand Up @@ -503,7 +513,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws
urLedgerMgr.setLostBookieRecoveryDelay(50);

// shutdown a non auditor bookie; choosing non-auditor to avoid another election
String shutdownBookie = shutDownNonAuditorBookie();
AtomicReference<String> shutdownBookieRef = new AtomicReference<>();
CountDownLatch shutdownLatch = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie = shutDownNonAuditorBookie();
shutdownBookieRef.set(shutdownBookie);
shutdownLatch.countDown();
} catch (Exception ignore) {
}
}).start();

if (LOG.isDebugEnabled()) {
LOG.debug("Waiting for ledgers to be marked as under replicated");
Expand All @@ -522,9 +541,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws
urLedgerList.contains(ledgerId));
Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
String data = urLedgerData.get(ledgerId);
assertTrue("Bookie " + shutdownBookie
shutdownLatch.await();
assertTrue("Bookie " + shutdownBookieRef.get()
+ "is not listed in the ledger as missing replica :" + data,
data.contains(shutdownBookie));
data.contains(shutdownBookieRef.get()));
}

@Test
Expand All @@ -547,7 +567,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep
urLedgerMgr.setLostBookieRecoveryDelay(3);

// shutdown a non auditor bookie; choosing non-auditor to avoid another election
String shutdownBookie = shutDownNonAuditorBookie();
AtomicReference<String> shutdownBookieRef = new AtomicReference<>();
CountDownLatch shutdownLatch = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie = shutDownNonAuditorBookie();
shutdownBookieRef.set(shutdownBookie);
shutdownLatch.countDown();
} catch (Exception ignore) {
}
}).start();

if (LOG.isDebugEnabled()) {
LOG.debug("Waiting for ledgers to be marked as under replicated");
Expand All @@ -573,9 +602,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep
urLedgerList.contains(ledgerId));
Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
String data = urLedgerData.get(ledgerId);
assertTrue("Bookie " + shutdownBookie
shutdownLatch.await();
assertTrue("Bookie " + shutdownBookieRef.get()
+ "is not listed in the ledger as missing replica :" + data,
data.contains(shutdownBookie));
data.contains(shutdownBookieRef.get()));
}

@Test
Expand Down Expand Up @@ -664,7 +694,12 @@ public void testTriggerAuditorWithPendingAuditTask() throws Exception {
urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);

// shutdown a non auditor bookie; choosing non-auditor to avoid another election
String shutdownBookie = shutDownNonAuditorBookie();
new Thread(() -> {
try {
shutDownNonAuditorBookie();
} catch (Exception ignore) {
}
}).start();

if (LOG.isDebugEnabled()) {
LOG.debug("Waiting for ledgers to be marked as under replicated");
Expand Down Expand Up @@ -715,7 +750,12 @@ public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws
urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);

// shutdown a non auditor bookie; choosing non-auditor to avoid another election
String shutdownBookie = shutDownNonAuditorBookie();
new Thread(() -> {
try {
shutDownNonAuditorBookie();
} catch (Exception ignore) {
}
}).start();

if (LOG.isDebugEnabled()) {
LOG.debug("Waiting for ledgers to be marked as under replicated");
Expand Down Expand Up @@ -767,8 +807,17 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception {
// wait for 10 seconds before starting the recovery work when a bookie fails
urLedgerMgr.setLostBookieRecoveryDelay(10);

// shutdown a non auditor bookie to avoid an election
String shutdownBookie1 = shutDownNonAuditorBookie();
// shutdown a non auditor bookie; choosing non-auditor to avoid another election
AtomicReference<String> shutdownBookieRef1 = new AtomicReference<>();
CountDownLatch shutdownLatch1 = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie1 = shutDownNonAuditorBookie();
shutdownBookieRef1.set(shutdownBookie1);
shutdownLatch1.countDown();
} catch (Exception ignore) {
}
}).start();

// wait for 3 seconds and there shouldn't be any under replicated ledgers
// because we have delayed the start of audit by 10 seconds
Expand All @@ -780,7 +829,16 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception {
// the history about having delayed recovery remains. Hence we make sure
// we bring down a non auditor bookie. This should cause the audit to take
// place immediately and not wait for the remaining 7 seconds to elapse
String shutdownBookie2 = shutDownNonAuditorBookie();
AtomicReference<String> shutdownBookieRef2 = new AtomicReference<>();
CountDownLatch shutdownLatch2 = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie2 = shutDownNonAuditorBookie();
shutdownBookieRef2.set(shutdownBookie2);
shutdownLatch2.countDown();
} catch (Exception ignore) {
}
}).start();

// 2 second grace period for the ledgers to get reported as under replicated
Thread.sleep(2000);
Expand All @@ -793,9 +851,11 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception {
urLedgerList.contains(ledgerId));
Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
String data = urLedgerData.get(ledgerId);
assertTrue("Bookie " + shutdownBookie1 + shutdownBookie2
shutdownLatch1.await();
shutdownLatch2.await();
assertTrue("Bookie " + shutdownBookieRef1.get() + shutdownBookieRef2.get()
+ " are not listed in the ledger as missing replicas :" + data,
data.contains(shutdownBookie1) && data.contains(shutdownBookie2));
data.contains(shutdownBookieRef1.get()) && data.contains(shutdownBookieRef2.get()));
}

/**
Expand Down Expand Up @@ -825,7 +885,16 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception {
// shutdown a non auditor bookie to avoid an election
int idx1 = getShutDownNonAuditorBookieIdx("");
ServerConfiguration conf1 = confByIndex(idx1);
String shutdownBookie1 = shutdownBookie(idx1);
AtomicReference<String> shutdownBookieRef1 = new AtomicReference<>();
CountDownLatch shutdownLatch1 = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie1 = shutdownBookie(idx1);
shutdownBookieRef1.set(shutdownBookie1);
shutdownLatch1.countDown();
} catch (Exception ignore) {
}
}).start();

// wait for 2 seconds and there shouldn't be any under replicated ledgers
// because we have delayed the start of audit by 5 seconds
Expand All @@ -838,7 +907,17 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception {

// Now to simulate the rolling upgrade, bring down a bookie different from
// the one we brought down/up above.
String shutdownBookie2 = shutDownNonAuditorBookie(shutdownBookie1);
// shutdown a non auditor bookie; choosing non-auditor to avoid another election
AtomicReference<String> shutdownBookieRef2 = new AtomicReference<>();
CountDownLatch shutdownLatch2 = new CountDownLatch(1);
new Thread(() -> {
try {
String shutdownBookie2 = shutDownNonAuditorBookie();
shutdownBookieRef2.set(shutdownBookie2);
shutdownLatch2.countDown();
} catch (Exception ignore) {
}
}).start();

// since the first bookie that was brought down/up has come up, there is only
// one bookie down at this time. Hence the lost bookie check shouldn't start
Expand All @@ -856,11 +935,13 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception {
urLedgerList.contains(ledgerId));
Map<Long, String> urLedgerData = getUrLedgerData(urLedgerList);
String data = urLedgerData.get(ledgerId);
assertTrue("Bookie " + shutdownBookie1 + "wrongly listed as missing the ledger: " + data,
!data.contains(shutdownBookie1));
assertTrue("Bookie " + shutdownBookie2
shutdownLatch1.await();
shutdownLatch2.await();
assertTrue("Bookie " + shutdownBookieRef1.get() + "wrongly listed as missing the ledger: " + data,
!data.contains(shutdownBookieRef1.get()));
assertTrue("Bookie " + shutdownBookieRef2.get()
+ " is not listed in the ledger as missing replicas :" + data,
data.contains(shutdownBookie2));
data.contains(shutdownBookieRef2.get()));
LOG.info("*****************Test Complete");
}

Expand Down

0 comments on commit e764366

Please sign in to comment.