Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix force GC doesn't work under forceAllowCompaction when disk is full #3205

Merged
merged 6 commits into from
Apr 20, 2022
Merged

Fix force GC doesn't work under forceAllowCompaction when disk is full #3205

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
650fa8f
fix force gc doesn't work under forceAllowCompaction when disk is full
hangc0276 Apr 13, 2022
868440d
add unit test
hangc0276 Apr 14, 2022
1d1e262
add parameter for rest api to trigger compaction
hangc0276 Apr 14, 2022
ecc1386
fix unit test
hangc0276 Apr 14, 2022
0ba3974
address comments
hangc0276 Apr 14, 2022
6e256cc
address comments
hangc0276 Apr 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,15 @@ public void enableForceGC() {
}
}

public void enableForceGC(Boolean forceMajor, Boolean forceMinor) {
if (forceGarbageCollection.compareAndSet(false, true)) {
LOG.info("Forced garbage collection triggered by thread: {}, forceMajor: {}, forceMinor: {}",
Thread.currentThread().getName(), forceMajor, forceMinor);
triggerGC(true, forceMajor == null ? suspendMajorCompaction.get() : !forceMajor,
forceMinor == null ? suspendMinorCompaction.get() : !forceMinor);
}
}

public void disableForceGC() {
if (forceGarbageCollection.compareAndSet(true, false)) {
LOG.info("{} disabled force garbage collection since bookie has enough space now.", Thread
Expand Down
5 changes: 5 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,11 @@ public void forceGC() {
gcThread.enableForceGC();
}

@Override
public void forceGC(Boolean forceMajor, Boolean forceMinor) {
gcThread.enableForceGC(forceMajor, forceMinor);
}

@Override
public boolean isInForceGC() {
return gcThread.isInForceGC();
Expand Down
7 changes: 7 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,13 @@ default void forceGC() {
return;
}

/**
* Force trigger Garbage Collection with forceMajor or forceMinor parameter.
*/
default void forceGC(Boolean forceMajor, Boolean forceMinor) {
return;
}

/**
* Class for describing location of a generic inconsistency. Implementations should
* ensure that detail is populated with an exception which adequately describes the
Expand Down
5 changes: 5 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,11 @@ public void forceGC() {
interleavedLedgerStorage.forceGC();
}

@Override
public void forceGC(Boolean forceMajor, Boolean forceMinor) {
interleavedLedgerStorage.forceGC(forceMajor, forceMinor);
}

@Override
public List<DetectedInconsistency> localConsistencyCheck(Optional<RateLimiter> rateLimiter) throws IOException {
return interleavedLedgerStorage.localConsistencyCheck(rateLimiter);
Expand Down
5 changes: 5 additions & 0 deletions ...keeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,11 @@ public void forceGC() {
ledgerStorageList.stream().forEach(SingleDirectoryDbLedgerStorage::forceGC);
}

@Override
public void forceGC(Boolean forceMajor, Boolean forceMinor) {
ledgerStorageList.stream().forEach(s -> s.forceGC(forceMajor, forceMinor));
}

@Override
public boolean isInForceGC() {
return ledgerStorageList.stream().anyMatch(SingleDirectoryDbLedgerStorage::isInForceGC);
Expand Down
5 changes: 5 additions & 0 deletions ...rc/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,11 @@ public void forceGC() {
gcThread.enableForceGC();
}

@Override
public void forceGC(Boolean forceMajor, Boolean forceMinor) {
gcThread.enableForceGC(forceMajor, forceMinor);
}

@Override
public boolean isInForceGC() {
return gcThread.isInForceGC();
Expand Down
13 changes: 12 additions & 1 deletion ...eper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

import static com.google.common.base.Preconditions.checkNotNull;

import java.util.HashMap;
import java.util.Map;
import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
Expand Down Expand Up @@ -61,7 +63,16 @@ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
HttpServiceResponse response = new HttpServiceResponse();

if (HttpServer.Method.PUT == request.getMethod()) {
bookieServer.getBookie().getLedgerStorage().forceGC();
String requestBody = request.getBody();
if (null == requestBody) {
bookieServer.getBookie().getLedgerStorage().forceGC();
} else {
@SuppressWarnings("unchecked")
Map<String, Object> configMap = JsonUtil.fromJson(requestBody, HashMap.class);
Boolean forceMajor = (Boolean) configMap.getOrDefault("forceMajor", null);
Boolean forceMinor = (Boolean) configMap.getOrDefault("forceMinor", null);
bookieServer.getBookie().getLedgerStorage().forceGC(forceMajor, forceMinor);
}

String output = "Triggered GC on BookieServer: " + bookieServer.toString();
String jsonResponse = JsonUtil.toJson(output);
Expand Down
89 changes: 89 additions & 0 deletions bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,95 @@ public void checkpointComplete(Checkpoint checkPoint, boolean compact)
});
}

@Test
public void testForceGarbageCollectionWhenDiskIsFull() throws Exception {
testForceGarbageCollectionWhenDiskIsFull(true);
testForceGarbageCollectionWhenDiskIsFull(false);
}

public void testForceGarbageCollectionWhenDiskIsFull(boolean isForceCompactionAllowWhenDisableCompaction)
throws Exception {

restartBookies(conf -> {
if (isForceCompactionAllowWhenDisableCompaction) {
conf.setMinorCompactionInterval(0);
conf.setMajorCompactionInterval(0);
conf.setForceAllowCompaction(true);
conf.setMajorCompactionThreshold(0.5f);
conf.setMinorCompactionThreshold(0.2f);
} else {
conf.setMinorCompactionInterval(120000);
nicoloboschi marked this conversation as resolved.
Show resolved Hide resolved
conf.setMajorCompactionInterval(240000);
}
return conf;
});

getGCThread().suspendMajorGC();
getGCThread().suspendMinorGC();
long majorCompactionCntBeforeGC = 0;
long minorCompactionCntBeforeGC = 0;
long majorCompactionCntAfterGC = 0;
long minorCompactionCntAfterGC = 0;

// disable forceMajor and forceMinor
majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
getGCThread().triggerGC(true, true, true).get();
majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
if (isForceCompactionAllowWhenDisableCompaction) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the difference between the if and else? Looks like they are asserting the same thing?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whether we will enter major or minor compaction is controlled by

((isForceMajorCompactionAllow && force) || (enableMajorCompaction
                    && (force || curTime - lastMajorCompactionTime > majorCompactionInterval)))
                    && (!suspendMajor)

That is:

  1. isForceMajorCompactionAllow && force or (enableMajorCompaction && (force || curTime - lastMajorCompactionTime > majorCompactionInterval)
  2. suspendMajor

We must meet above both conditions.

The if and else is different code path controlled by isForceMajorCompactionAllow && force and (enableMajorCompaction && (force || curTime - lastMajorCompactionTime > majorCompactionInterval)

assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
} else {
assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
}

// enable forceMajor and forceMinor
majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
getGCThread().triggerGC(true, false, false).get();
majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();

if (isForceCompactionAllowWhenDisableCompaction) {
assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
} else {
assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
}

// enable forceMajor and disable forceMinor
majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
getGCThread().triggerGC(true, false, true).get();
majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
if (isForceCompactionAllowWhenDisableCompaction) {
assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
} else {
assertEquals(majorCompactionCntBeforeGC + 1, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC, minorCompactionCntAfterGC);
}

// disable forceMajor and enable forceMinor
majorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntBeforeGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
getGCThread().triggerGC(true, true, false).get();
majorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMajorCompactionCounter();
minorCompactionCntAfterGC = getGCThread().getGarbageCollectionStatus().getMinorCompactionCounter();
if (isForceCompactionAllowWhenDisableCompaction) {
assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC + 1, minorCompactionCntAfterGC);
} else {
assertEquals(majorCompactionCntBeforeGC, majorCompactionCntAfterGC);
assertEquals(minorCompactionCntBeforeGC + 1, minorCompactionCntAfterGC);
}

}

@Test
public void testMinorCompaction() throws Exception {
// prepare data
Expand Down
5 changes: 5 additions & 0 deletions bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/MockLedgerStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,11 @@ public void forceGC() {
LedgerStorage.super.forceGC();
}

@Override
public void forceGC(Boolean forceMajor, Boolean forceMinor) {
LedgerStorage.super.forceGC(forceMajor, forceMinor);
}

@Override
public List<DetectedInconsistency> localConsistencyCheck(Optional<RateLimiter> rateLimiter) throws IOException {
return LedgerStorage.super.localConsistencyCheck(rateLimiter);
Expand Down