From ac6b13d5e06968e1a9d73a6e6d82fe241aea7b13 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sun, 25 Aug 2024 11:04:01 +0800 Subject: [PATCH 01/57] extract the adaptive lock --- .../store/PutMessageReentrantLock.java | 4 +- .../store/config/MessageStoreConfig.java | 26 ++++++ .../rocketmq/store/lock/AdaptiveLock.java | 32 +++++++ .../rocketmq/store/lock/AdaptiveLockImpl.java | 85 +++++++++++++++++++ .../store/lock/CollisionRetreatLock.java | 63 ++++++++++++++ 5 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java index 9aa80d86721..88d8124c5e3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java @@ -16,12 +16,14 @@ */ package org.apache.rocketmq.store; +import org.apache.rocketmq.store.lock.AdaptiveLock; + import java.util.concurrent.locks.ReentrantLock; /** * Exclusive lock implementation to put message */ -public class PutMessageReentrantLock implements PutMessageLock { +public class PutMessageReentrantLock implements PutMessageLock, AdaptiveLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0b45d92418e..443c238db3e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -424,6 +424,17 @@ public class MessageStoreConfig { private boolean putConsumeQueueDataByFileChannel = true; + /** + * Spin number in the retreat strategy of spin lock + * Default is 1000 + */ + private int spinLockCollisionRetreatOptimalDegree = 1000; + + /* + *Critical TPS that triggers the adaptive locking mechanism switch + */ + private int tpsSwapCriticalPoint = 50000; + public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } @@ -1854,4 +1865,19 @@ public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocks this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; } + public int getSpinLockCollisionRetreatOptimalDegree() { + return spinLockCollisionRetreatOptimalDegree; + } + + public void setSpinLockCollisionRetreatOptimalDegree(int optimalDegree) { + this.spinLockCollisionRetreatOptimalDegree = optimalDegree; + } + + public int getTpsSwapCriticalPoint() { + return tpsSwapCriticalPoint; + } + + public void setTpsSwapCriticalPoint(int tpsSwapCriticalPoint) { + this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java new file mode 100644 index 00000000000..5ae0bbe2ef6 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.config.MessageStoreConfig; + +public interface AdaptiveLock { + + void lock(); + + void unlock(); + + default void update(MessageStoreConfig messageStoreConfig) { + } + + default void swap() { + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java new file mode 100644 index 00000000000..98df7a818ab --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.PutMessageReentrantLock; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class AdaptiveLockImpl implements AdaptiveLock { + + private AdaptiveLock adaptiveLock; + + //state + private AtomicBoolean state = new AtomicBoolean(true); + + private List locks; + + public AdaptiveLockImpl() { + this.locks = new LinkedList<>(); + this.locks.add(new CollisionRetreatLock()); + this.locks.add(new PutMessageReentrantLock()); + adaptiveLock = new CollisionRetreatLock(); + } + + @Override + public void lock() { + while (this.state.get()) { + this.adaptiveLock.lock(); + } + } + + @Override + public void unlock() { + while (this.state.get()) { + this.adaptiveLock.unlock(); + } + } + + @Override + public void update(MessageStoreConfig messageStoreConfig) { + this.adaptiveLock.update(messageStoreConfig); + } + + @Override + public void swap() { + if (this.state.compareAndSet(true, false)) { + //change adaptiveLock + + this.state.compareAndSet(false, true); + } + } + + public List getLocks() { + return this.locks; + } + + public void setLocks(List locks) { + this.locks = locks; + } + + public boolean getState() { + return this.state.get(); + } + + public void setState(boolean state) { + this.state.set(state); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java new file mode 100644 index 00000000000..6661299c0a1 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.config.MessageStoreConfig; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class CollisionRetreatLock implements AdaptiveLock { + + private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); + + private int optimalDegree; + + public CollisionRetreatLock() { + this.optimalDegree = 1000; + } + + public CollisionRetreatLock(int optimalDegree) { + this.optimalDegree = optimalDegree; + } + + @Override + public void lock() { + int spinDegree = this.optimalDegree; + while (true) { + for (int i = 0; i < spinDegree; i++) { + if (this.putMessageSpinLock.compareAndSet(true, false)) { + return; + } + } + Thread.yield(); + } + } + + @Override + public void unlock() { + this.putMessageSpinLock.compareAndSet(false, true); + } + + @Override + public void update(MessageStoreConfig messageStoreConfig) { + this.optimalDegree = messageStoreConfig.getSpinLockCollisionRetreatOptimalDegree(); + } + + public int getOptimalDegree() { + return this.optimalDegree; + } +} From c4bdf516e667fe41abf127e0f1b5a1793168d95a Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sun, 25 Aug 2024 11:27:41 +0800 Subject: [PATCH 02/57] extract the adaptive lock --- .../main/java/org/apache/rocketmq/store/CommitLog.java | 6 ++++-- .../org/apache/rocketmq/store/lock/AdaptiveLockImpl.java | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index f707d8fbd87..2ad99847d66 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,6 +60,8 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; +import org.apache.rocketmq.store.lock.AdaptiveLock; +import org.apache.rocketmq.store.lock.AdaptiveLockImpl; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -92,7 +94,7 @@ public class CommitLog implements Swappable { private volatile long beginTimeInLock = 0; - protected final PutMessageLock putMessageLock; + protected final AdaptiveLock putMessageLock; protected final TopicQueueLock topicQueueLock; @@ -129,7 +131,7 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); + this.putMessageLock = new AdaptiveLockImpl(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 98df7a818ab..18c01cae172 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -41,16 +41,16 @@ public AdaptiveLockImpl() { @Override public void lock() { - while (this.state.get()) { - this.adaptiveLock.lock(); + while (!this.state.get()) { } + this.adaptiveLock.lock(); } @Override public void unlock() { - while (this.state.get()) { - this.adaptiveLock.unlock(); + while (!this.state.get()) { } + this.adaptiveLock.unlock(); } @Override From 12094b4bd3538adb8efa1fa3a2082e88c4a77a66 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sat, 31 Aug 2024 17:12:49 +0800 Subject: [PATCH 03/57] feat(): perfect the adaptive lock --- .../rocketmq/store/lock/AdaptiveLockImpl.java | 85 +++++++++++++++---- .../rocketmq/store/lock/AdaptiveLockTest.java | 65 ++++++++++++++ 2 files changed, 134 insertions(+), 16 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 18c01cae172..c738f0e2295 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -19,38 +19,56 @@ import org.apache.rocketmq.store.PutMessageReentrantLock; import org.apache.rocketmq.store.config.MessageStoreConfig; -import java.util.LinkedList; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class AdaptiveLockImpl implements AdaptiveLock { - private AdaptiveLock adaptiveLock; //state private AtomicBoolean state = new AtomicBoolean(true); - private List locks; + private Map locks; + + private final List tpsTable; public AdaptiveLockImpl() { - this.locks = new LinkedList<>(); - this.locks.add(new CollisionRetreatLock()); - this.locks.add(new PutMessageReentrantLock()); - adaptiveLock = new CollisionRetreatLock(); + this.locks = new HashMap<>(); + this.locks.put("Reentrant", new PutMessageReentrantLock()); + this.locks.put("Collision", new CollisionRetreatLock()); + + this.tpsTable = new ArrayList<>(2); + this.tpsTable.add(new AtomicInteger(0)); + this.tpsTable.add(new AtomicInteger(0)); + + adaptiveLock = this.locks.get("Collision"); } @Override public void lock() { - while (!this.state.get()) { - } + tpsTable.get(LocalTime.now().getSecond() % 2).getAndIncrement(); + boolean state; + do { + state = this.state.get(); + } while (!state); + this.adaptiveLock.lock(); } @Override public void unlock() { - while (!this.state.get()) { - } + boolean state; + do { + state = this.state.get(); + } while (!state); + this.adaptiveLock.unlock(); + swap(); } @Override @@ -60,18 +78,45 @@ public void update(MessageStoreConfig messageStoreConfig) { @Override public void swap() { - if (this.state.compareAndSet(true, false)) { - //change adaptiveLock + boolean needSwap = false; + int slot = LocalTime.now().getSecond() % 2 - 1 >= 0 ? 0 : 1; + int tps = this.tpsTable.get(slot).get(); + this.tpsTable.get(slot).set(-1); + if (tps == -1) { + return; + } + if (tps > 50000) { + if (this.adaptiveLock instanceof CollisionRetreatLock) { + needSwap = true; + } + } else { + if (this.adaptiveLock instanceof PutMessageReentrantLock) { + needSwap = true; + } + } - this.state.compareAndSet(false, true); + if (needSwap) { + if (this.state.compareAndSet(true, false)) { + this.adaptiveLock.lock(); + if (this.adaptiveLock instanceof CollisionRetreatLock) { + this.adaptiveLock = this.locks.get("Reentrant"); + } else { + this.adaptiveLock = this.locks.get("Collision"); + } + try { + this.adaptiveLock.unlock(); + } catch (Exception ignore){ + } + this.state.compareAndSet(false, true); + } } } public List getLocks() { - return this.locks; + return (List) this.locks.values(); } - public void setLocks(List locks) { + public void setLocks(Map locks) { this.locks = locks; } @@ -82,4 +127,12 @@ public boolean getState() { public void setState(boolean state) { this.state.set(state); } + + public AdaptiveLock getAdaptiveLock() { + return this.adaptiveLock; + } + + public List getTpsTable() { + return this.tpsTable; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java new file mode 100644 index 00000000000..775ef1309c7 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.PutMessageReentrantLock; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class AdaptiveLockTest { + + AdaptiveLockImpl adaptiveLock; + + @Before + public void init() { + adaptiveLock = new AdaptiveLockImpl(); + + } + + @Test + public void testAdaptiveLock() throws InterruptedException { + assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + for (int i = 0; i < 100000; i++) { + adaptiveLock.lock(); + adaptiveLock.unlock(); + if (i == 50000) Thread.sleep(1000); + } + assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); + System.out.println(adaptiveLock.getTpsTable()); + + Thread.sleep(1000L); + adaptiveLock.lock(); + adaptiveLock.unlock(); + for (int i = 0; i < 300; i++) { + adaptiveLock.lock(); + adaptiveLock.unlock(); + Thread.sleep(10); + } + assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + System.out.println(adaptiveLock.getTpsTable()); + + for (int i = 0; i < 100000; i++) { + adaptiveLock.lock(); + adaptiveLock.unlock(); + if (i == 50000) Thread.sleep(1000); + } + assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); + + } +} From 2b3c6473917e8e3b65eed646618657af68a4c824 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sat, 31 Aug 2024 17:16:10 +0800 Subject: [PATCH 04/57] feat(): perfect the adaptive lock --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java | 2 +- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index c738f0e2295..2c29effbe81 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -105,7 +105,7 @@ public void swap() { } try { this.adaptiveLock.unlock(); - } catch (Exception ignore){ + } catch (Exception ignore) { } this.state.compareAndSet(false, true); } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 775ef1309c7..acd8fb58d30 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -41,7 +41,6 @@ public void testAdaptiveLock() throws InterruptedException { if (i == 50000) Thread.sleep(1000); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); - System.out.println(adaptiveLock.getTpsTable()); Thread.sleep(1000L); adaptiveLock.lock(); @@ -52,7 +51,6 @@ public void testAdaptiveLock() throws InterruptedException { Thread.sleep(10); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); - System.out.println(adaptiveLock.getTpsTable()); for (int i = 0; i < 100000; i++) { adaptiveLock.lock(); From 6d02e7e92213419a2dd0cc120926d3a2a88baf61 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sat, 31 Aug 2024 20:24:49 +0800 Subject: [PATCH 05/57] Optimized code type --- .../main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- .../org/apache/rocketmq/store/lock/AdaptiveLockImpl.java | 9 +++++++-- .../org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 5 ++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 2ad99847d66..61022104a1f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -131,7 +131,7 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = new AdaptiveLockImpl(); + this.putMessageLock = new AdaptiveLockImpl(messageStore.getMessageStoreConfig().getTpsSwapCriticalPoint()); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 2c29effbe81..48ea871827f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -37,7 +37,9 @@ public class AdaptiveLockImpl implements AdaptiveLock { private final List tpsTable; - public AdaptiveLockImpl() { + private static int TPS_SWAP_CRITICAL_POINT; + + public AdaptiveLockImpl(int tpsSwapCriticalPoint) { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); this.locks.put("Collision", new CollisionRetreatLock()); @@ -46,6 +48,7 @@ public AdaptiveLockImpl() { this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); + TPS_SWAP_CRITICAL_POINT = tpsSwapCriticalPoint; adaptiveLock = this.locks.get("Collision"); } @@ -85,7 +88,7 @@ public void swap() { if (tps == -1) { return; } - if (tps > 50000) { + if (tps > TPS_SWAP_CRITICAL_POINT) { if (this.adaptiveLock instanceof CollisionRetreatLock) { needSwap = true; } @@ -106,6 +109,8 @@ public void swap() { try { this.adaptiveLock.unlock(); } catch (Exception ignore) { + //Used to synchronize the lock state, + //ReentrantLock throws an exception when unlock is executed when the lock is free, so it is caught and ignored } this.state.compareAndSet(false, true); } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index acd8fb58d30..f84a45000ca 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -28,13 +28,13 @@ public class AdaptiveLockTest { @Before public void init() { - adaptiveLock = new AdaptiveLockImpl(); - + adaptiveLock = new AdaptiveLockImpl(50000); } @Test public void testAdaptiveLock() throws InterruptedException { assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + for (int i = 0; i < 100000; i++) { adaptiveLock.lock(); adaptiveLock.unlock(); @@ -58,6 +58,5 @@ public void testAdaptiveLock() throws InterruptedException { if (i == 50000) Thread.sleep(1000); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); - } } From 2a5e23ea639f81f1a02843b81db4e017efe3d4e7 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sat, 31 Aug 2024 20:43:23 +0800 Subject: [PATCH 06/57] Optimized code type --- .../org/apache/rocketmq/store/lock/AdaptiveLockImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 48ea871827f..53209c5e59f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -37,7 +37,7 @@ public class AdaptiveLockImpl implements AdaptiveLock { private final List tpsTable; - private static int TPS_SWAP_CRITICAL_POINT; + private int tpsSwapCriticalPoint; public AdaptiveLockImpl(int tpsSwapCriticalPoint) { this.locks = new HashMap<>(); @@ -48,7 +48,7 @@ public AdaptiveLockImpl(int tpsSwapCriticalPoint) { this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); - TPS_SWAP_CRITICAL_POINT = tpsSwapCriticalPoint; + this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; adaptiveLock = this.locks.get("Collision"); } @@ -88,7 +88,7 @@ public void swap() { if (tps == -1) { return; } - if (tps > TPS_SWAP_CRITICAL_POINT) { + if (tps > this.tpsSwapCriticalPoint) { if (this.adaptiveLock instanceof CollisionRetreatLock) { needSwap = true; } From 9714ea0aab1f5cee5944f1541f6653dbc8622ea5 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sun, 1 Sep 2024 10:27:20 +0800 Subject: [PATCH 07/57] Optimized code type --- .../apache/rocketmq/store/lock/AdaptiveLockImpl.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 53209c5e59f..49cb12aa049 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -134,10 +134,18 @@ public void setState(boolean state) { } public AdaptiveLock getAdaptiveLock() { - return this.adaptiveLock; + return adaptiveLock; } public List getTpsTable() { - return this.tpsTable; + return tpsTable; + } + + public void setTpsSwapCriticalPoint(int tpsSwapCriticalPoint) { + this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; + } + + public int getTpsSwapCriticalPoint() { + return tpsSwapCriticalPoint; } } From ff9f994a06098573157449bc0e6a32ba36731d48 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan <3424672656@qq.com> Date: Sun, 1 Sep 2024 12:17:21 +0800 Subject: [PATCH 08/57] fix fail test --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index f84a45000ca..e625eaf43e7 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -38,7 +38,7 @@ public void testAdaptiveLock() throws InterruptedException { for (int i = 0; i < 100000; i++) { adaptiveLock.lock(); adaptiveLock.unlock(); - if (i == 50000) Thread.sleep(1000); + if (i == 70000) Thread.sleep(1000); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); @@ -55,7 +55,7 @@ public void testAdaptiveLock() throws InterruptedException { for (int i = 0; i < 100000; i++) { adaptiveLock.lock(); adaptiveLock.unlock(); - if (i == 50000) Thread.sleep(1000); + if (i == 70000) Thread.sleep(1000); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); } From 439e2b8d3460fbaf18947af5b1a84320032806d2 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 20 Sep 2024 15:59:23 +0800 Subject: [PATCH 09/57] Optimize the adaptive locking mechanism logic --- .../java/org/apache/rocketmq/store/CommitLog.java | 2 +- .../apache/rocketmq/store/lock/AdaptiveLockImpl.java | 11 ++++++----- .../apache/rocketmq/store/lock/AdaptiveLockTest.java | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 61022104a1f..2ad99847d66 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -131,7 +131,7 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = new AdaptiveLockImpl(messageStore.getMessageStoreConfig().getTpsSwapCriticalPoint()); + this.putMessageLock = new AdaptiveLockImpl(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 49cb12aa049..f5553306fae 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -39,7 +39,7 @@ public class AdaptiveLockImpl implements AdaptiveLock { private int tpsSwapCriticalPoint; - public AdaptiveLockImpl(int tpsSwapCriticalPoint) { + public AdaptiveLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); this.locks.put("Collision", new CollisionRetreatLock()); @@ -48,7 +48,6 @@ public AdaptiveLockImpl(int tpsSwapCriticalPoint) { this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); - this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; adaptiveLock = this.locks.get("Collision"); } @@ -88,12 +87,14 @@ public void swap() { if (tps == -1) { return; } - if (tps > this.tpsSwapCriticalPoint) { - if (this.adaptiveLock instanceof CollisionRetreatLock) { + + if (this.adaptiveLock instanceof CollisionRetreatLock) { + if () { + this.tpsSwapCriticalPoint = tps; needSwap = true; } } else { - if (this.adaptiveLock instanceof PutMessageReentrantLock) { + if (tps < this.tpsSwapCriticalPoint) { needSwap = true; } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index e625eaf43e7..948041d3fcb 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -28,7 +28,7 @@ public class AdaptiveLockTest { @Before public void init() { - adaptiveLock = new AdaptiveLockImpl(50000); + adaptiveLock = new AdaptiveLockImpl(); } @Test From 1d004042b9a3616a04f22460d43d8e36596d2e55 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 20 Sep 2024 16:37:50 +0800 Subject: [PATCH 10/57] Optimize the adaptive locking mechanism logic --- .../rocketmq/store/lock/AdaptiveLockImpl.java | 12 +++++-- .../store/lock/CollisionRetreatLock.java | 36 +++++++++++++++++-- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index f5553306fae..0912b9636ba 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -89,10 +89,16 @@ public void swap() { } if (this.adaptiveLock instanceof CollisionRetreatLock) { - if () { - this.tpsSwapCriticalPoint = tps; - needSwap = true; + CollisionRetreatLock lock = (CollisionRetreatLock) this.adaptiveLock; + if (lock.getNumberOfRetreat(slot) / tps < 1 / 5) { + if (lock.isAdapt()) { + lock.adapt(); + } else { + this.tpsSwapCriticalPoint = tps; + needSwap = true; + } } + lock.setNumberOfRetreat(0, slot); } else { if (tps < this.tpsSwapCriticalPoint) { needSwap = true; diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java index 6661299c0a1..02da71e61f5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java @@ -18,7 +18,11 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class CollisionRetreatLock implements AdaptiveLock { @@ -26,12 +30,17 @@ public class CollisionRetreatLock implements AdaptiveLock { private int optimalDegree; + private int maxOptimalDegree; + + private List numberOfRetreat; + public CollisionRetreatLock() { this.optimalDegree = 1000; - } + this.maxOptimalDegree = 10000; - public CollisionRetreatLock(int optimalDegree) { - this.optimalDegree = optimalDegree; + this.numberOfRetreat = new ArrayList<>(2); + this.numberOfRetreat.add(new AtomicInteger(0)); + this.numberOfRetreat.add(new AtomicInteger(0)); } @Override @@ -43,6 +52,7 @@ public void lock() { return; } } + this.numberOfRetreat.get(LocalTime.now().getSecond() % 2).incrementAndGet(); Thread.yield(); } } @@ -60,4 +70,24 @@ public void update(MessageStoreConfig messageStoreConfig) { public int getOptimalDegree() { return this.optimalDegree; } + + public void setOptimalDegree(int optimalDegree) { + this.optimalDegree = optimalDegree; + } + + public int getNumberOfRetreat(int pos) { + return numberOfRetreat.get(pos).get(); + } + + public void setNumberOfRetreat(int numberOfRetreat, int pos) { + this.numberOfRetreat.get(pos).set(numberOfRetreat); + } + + public boolean isAdapt() { + return this.optimalDegree < this.maxOptimalDegree; + } + + public void adapt() { + //TODO adapt + } } From 0904c5031571b22635b13378b0f479a528cc21bf Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Sat, 21 Sep 2024 11:02:10 +0800 Subject: [PATCH 11/57] feat:Adaptive locking mechanism adjustment --- .../rocketmq/store/lock/AdaptiveLockImpl.java | 14 +++- .../store/lock/CollisionRetreatLock.java | 65 +++++++++++++++++-- .../rocketmq/store/lock/AdaptiveLockTest.java | 64 ++++++++++++++---- 3 files changed, 122 insertions(+), 21 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index f5553306fae..f468e1a21bf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -89,10 +89,18 @@ public void swap() { } if (this.adaptiveLock instanceof CollisionRetreatLock) { - if () { - this.tpsSwapCriticalPoint = tps; - needSwap = true; + CollisionRetreatLock lock = (CollisionRetreatLock) this.adaptiveLock; + if (lock.getNumberOfRetreat(slot) * 5 >= tps) { + if (lock.isAdapt()) { + lock.adapt(true); + } else { + this.tpsSwapCriticalPoint = tps; + needSwap = true; + } + } else if (lock.getNumberOfRetreat(slot) * 25 <= tps) { + lock.adapt(false); } + lock.setNumberOfRetreat(slot, 0); } else { if (tps < this.tpsSwapCriticalPoint) { needSwap = true; diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java index 6661299c0a1..12d81eb06fb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java @@ -18,7 +18,11 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class CollisionRetreatLock implements AdaptiveLock { @@ -26,12 +30,20 @@ public class CollisionRetreatLock implements AdaptiveLock { private int optimalDegree; + private int initOptimalDegree; + + private int maxOptimalDegree; + + private final List numberOfRetreat; + public CollisionRetreatLock() { - this.optimalDegree = 1000; - } + this.initOptimalDegree = 1000; + this.maxOptimalDegree = 10000; + this.optimalDegree = initOptimalDegree; - public CollisionRetreatLock(int optimalDegree) { - this.optimalDegree = optimalDegree; + numberOfRetreat = new ArrayList<>(2); + numberOfRetreat.add(new AtomicInteger(0)); + numberOfRetreat.add(new AtomicInteger(0)); } @Override @@ -43,6 +55,7 @@ public void lock() { return; } } + numberOfRetreat.get(LocalTime.now().getSecond() % 2).getAndIncrement(); Thread.yield(); } } @@ -60,4 +73,48 @@ public void update(MessageStoreConfig messageStoreConfig) { public int getOptimalDegree() { return this.optimalDegree; } + + public void setOptimalDegree(int optimalDegree) { + this.optimalDegree = optimalDegree; + } + + public int getMaxOptimalDegree() { + return maxOptimalDegree; + } + + public void setMaxOptimalDegree() { + this.maxOptimalDegree = maxOptimalDegree; + } + + public boolean isAdapt() { + return optimalDegree < maxOptimalDegree; + } + + public synchronized void adapt(boolean isRise) { + if (isRise) { + if (optimalDegree * 2 <= maxOptimalDegree) { + optimalDegree *= 2; + } else { + if (optimalDegree + initOptimalDegree <= maxOptimalDegree) { + optimalDegree += initOptimalDegree; + } + } + } else { + if (optimalDegree / 2 >= initOptimalDegree) { + optimalDegree /= 2; + } else { + if (optimalDegree > 2 * initOptimalDegree) { + optimalDegree -= initOptimalDegree; + } + } + } + } + + public int getNumberOfRetreat(int pos) { + return numberOfRetreat.get(pos).get(); + } + + public void setNumberOfRetreat(int pos, int size) { + this.numberOfRetreat.get(pos).set(size); + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 948041d3fcb..2a2df0def5d 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -20,6 +20,7 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class AdaptiveLockTest { @@ -35,28 +36,63 @@ public void init() { public void testAdaptiveLock() throws InterruptedException { assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); - for (int i = 0; i < 100000; i++) { + adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + adaptiveLock.unlock(); + } + }).start(); + Thread.sleep(1000); + adaptiveLock.unlock(); + assertEquals(2000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); + + adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + adaptiveLock.unlock(); + } + }).start(); + Thread.sleep(1000); + adaptiveLock.unlock(); + assertEquals(4000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); + + for (int i = 0; i < 3; i++) { adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + adaptiveLock.unlock(); + } + }).start(); + Thread.sleep(1000); adaptiveLock.unlock(); - if (i == 70000) Thread.sleep(1000); } + assertEquals(10000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); + + + adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + adaptiveLock.unlock(); + } + }).start(); + Thread.sleep(1000); + adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); Thread.sleep(1000L); adaptiveLock.lock(); adaptiveLock.unlock(); - for (int i = 0; i < 300; i++) { - adaptiveLock.lock(); - adaptiveLock.unlock(); - Thread.sleep(10); - } + Thread.sleep(1000L); + adaptiveLock.lock(); + adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); - - for (int i = 0; i < 100000; i++) { - adaptiveLock.lock(); - adaptiveLock.unlock(); - if (i == 70000) Thread.sleep(1000); - } - assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); } } From 99e07d67fa4bc39ebe5dd5cbf69d8fb76434f4a5 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Sat, 21 Sep 2024 11:09:24 +0800 Subject: [PATCH 12/57] feat:Adaptive locking mechanism adjustment --- .../org/apache/rocketmq/store/lock/CollisionRetreatLock.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java index 12d81eb06fb..b7c1cfc64d0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java @@ -82,7 +82,7 @@ public int getMaxOptimalDegree() { return maxOptimalDegree; } - public void setMaxOptimalDegree() { + public void setMaxOptimalDegree(int maxOptimalDegree) { this.maxOptimalDegree = maxOptimalDegree; } From ef10b0a82060df19b35425028ca5a888043fcf9d Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Sat, 21 Sep 2024 11:56:09 +0800 Subject: [PATCH 13/57] feat:Adaptive locking mechanism adjustment --- .../rocketmq/store/lock/AdaptiveLockTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 2a2df0def5d..172ef184d32 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -94,5 +94,26 @@ public void run() { adaptiveLock.lock(); adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + + for (int i = 0; i < 10; i++) { + new Thread(new Runnable() { + @Override + public void run() { + for (int i = 0; i < 100000; i++) { + adaptiveLock.lock(); + adaptiveLock.unlock(); + if (i == 50000) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // + } + } + } + + } + }).start(); + } + assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); } } From a49deda46247e2ff2d3c9d355f4ab72072e52c98 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 23 Sep 2024 15:42:13 +0800 Subject: [PATCH 14/57] Optimize the adaptive locking mechanism logic --- .../rocketmq/store/lock/AdaptiveLockImpl.java | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index f468e1a21bf..20f2a15d973 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -39,6 +39,8 @@ public class AdaptiveLockImpl implements AdaptiveLock { private int tpsSwapCriticalPoint; + private AtomicInteger currentThreadNum = new AtomicInteger(0); + public AdaptiveLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); @@ -59,17 +61,14 @@ public void lock() { state = this.state.get(); } while (!state); + currentThreadNum.incrementAndGet(); this.adaptiveLock.lock(); } @Override public void unlock() { - boolean state; - do { - state = this.state.get(); - } while (!state); - this.adaptiveLock.unlock(); + currentThreadNum.decrementAndGet(); swap(); } @@ -80,6 +79,9 @@ public void update(MessageStoreConfig messageStoreConfig) { @Override public void swap() { + if (!this.state.get()) { + return; + } boolean needSwap = false; int slot = LocalTime.now().getSecond() % 2 - 1 >= 0 ? 0 : 1; int tps = this.tpsTable.get(slot).get(); @@ -90,38 +92,43 @@ public void swap() { if (this.adaptiveLock instanceof CollisionRetreatLock) { CollisionRetreatLock lock = (CollisionRetreatLock) this.adaptiveLock; - if (lock.getNumberOfRetreat(slot) * 5 >= tps) { + int base = Math.min(200 + tps / 200, 500); + if (lock.getNumberOfRetreat(slot) * base >= tps) { if (lock.isAdapt()) { lock.adapt(true); } else { this.tpsSwapCriticalPoint = tps; needSwap = true; } - } else if (lock.getNumberOfRetreat(slot) * 25 <= tps) { + } else if (lock.getNumberOfRetreat(slot) * base * 3 / 2 <= tps) { lock.adapt(false); } lock.setNumberOfRetreat(slot, 0); } else { - if (tps < this.tpsSwapCriticalPoint) { + if (tps < this.tpsSwapCriticalPoint * 4 / 5) { needSwap = true; } } if (needSwap) { if (this.state.compareAndSet(true, false)) { - this.adaptiveLock.lock(); - if (this.adaptiveLock instanceof CollisionRetreatLock) { - this.adaptiveLock = this.locks.get("Reentrant"); - } else { - this.adaptiveLock = this.locks.get("Collision"); - } + int currentThreadNum; + do { + currentThreadNum = this.currentThreadNum.get(); + } while (currentThreadNum != 0); + try { - this.adaptiveLock.unlock(); - } catch (Exception ignore) { - //Used to synchronize the lock state, - //ReentrantLock throws an exception when unlock is executed when the lock is free, so it is caught and ignored + if (this.adaptiveLock instanceof CollisionRetreatLock) { + this.adaptiveLock = this.locks.get("Reentrant"); + } else { + this.adaptiveLock = this.locks.get("Collision"); + ((CollisionRetreatLock) this.adaptiveLock).adapt(false); + } + } catch (Exception e) { + //ignore + } finally { + this.state.compareAndSet(false, true); } - this.state.compareAndSet(false, true); } } } From 429f9e3158abea72bfdfc7948587bc3485228cc4 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 23 Sep 2024 16:19:26 +0800 Subject: [PATCH 15/57] Optimize the adaptive locking mechanism logic --- .../rocketmq/store/lock/AdaptiveLockImpl.java | 6 ++--- .../rocketmq/store/lock/AdaptiveLockTest.java | 27 ++----------------- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java index 20f2a15d973..2a5cff2870e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java @@ -84,9 +84,9 @@ public void swap() { } boolean needSwap = false; int slot = LocalTime.now().getSecond() % 2 - 1 >= 0 ? 0 : 1; - int tps = this.tpsTable.get(slot).get(); + int tps = this.tpsTable.get(slot).get() + 1; this.tpsTable.get(slot).set(-1); - if (tps == -1) { + if (tps == 0) { return; } @@ -105,7 +105,7 @@ public void swap() { } lock.setNumberOfRetreat(slot, 0); } else { - if (tps < this.tpsSwapCriticalPoint * 4 / 5) { + if (tps <= this.tpsSwapCriticalPoint * 4 / 5) { needSwap = true; } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 172ef184d32..e95f7eff713 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -87,33 +87,10 @@ public void run() { adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); - Thread.sleep(1000L); - adaptiveLock.lock(); - adaptiveLock.unlock(); - Thread.sleep(1000L); + adaptiveLock.lock(); + Thread.sleep(1000); adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); - - for (int i = 0; i < 10; i++) { - new Thread(new Runnable() { - @Override - public void run() { - for (int i = 0; i < 100000; i++) { - adaptiveLock.lock(); - adaptiveLock.unlock(); - if (i == 50000) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // - } - } - } - - } - }).start(); - } - assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); } } From 8b98e6643734bd116056a9fe75edac05581fe0a1 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Mon, 23 Sep 2024 16:42:58 +0800 Subject: [PATCH 16/57] Optimize the adaptive locking mechanism logic --- .../rocketmq/store/lock/AdaptiveLockTest.java | 27 ++----------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index e95f7eff713..f174c1c5c7c 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -48,19 +48,9 @@ public void run() { adaptiveLock.unlock(); assertEquals(2000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - adaptiveLock.lock(); - new Thread(new Runnable() { - @Override - public void run() { - adaptiveLock.lock(); - adaptiveLock.unlock(); - } - }).start(); - Thread.sleep(1000); - adaptiveLock.unlock(); - assertEquals(4000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - for (int i = 0; i < 3; i++) { + + for (int i = 0; i <= 8; i++) { adaptiveLock.lock(); new Thread(new Runnable() { @Override @@ -72,19 +62,6 @@ public void run() { Thread.sleep(1000); adaptiveLock.unlock(); } - assertEquals(10000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - - - adaptiveLock.lock(); - new Thread(new Runnable() { - @Override - public void run() { - adaptiveLock.lock(); - adaptiveLock.unlock(); - } - }).start(); - Thread.sleep(1000); - adaptiveLock.unlock(); assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); From 27a0a05a0fae96f18e8a6d2e4dee5638bddd3f66 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 15:51:00 +0800 Subject: [PATCH 17/57] feat:Supports the hot activation of ABS locks --- .../org/apache/rocketmq/store/CommitLog.java | 16 +++++-- .../store/PutMessageReentrantLock.java | 4 +- .../store/config/MessageStoreConfig.java | 16 +++---- ...tiveLock.java => AdaptiveBackOffLock.java} | 5 ++- ...Impl.java => AdaptiveBackOffLockImpl.java} | 45 +++++++++++++++---- .../store/lock/CollisionRetreatLock.java | 2 +- .../rocketmq/store/lock/AdaptiveLockTest.java | 4 +- 7 files changed, 65 insertions(+), 27 deletions(-) rename store/src/main/java/org/apache/rocketmq/store/lock/{AdaptiveLock.java => AdaptiveBackOffLock.java} (90%) rename store/src/main/java/org/apache/rocketmq/store/lock/{AdaptiveLockImpl.java => AdaptiveBackOffLockImpl.java} (78%) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 2ad99847d66..63614375ee7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,8 +60,9 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; -import org.apache.rocketmq.store.lock.AdaptiveLock; -import org.apache.rocketmq.store.lock.AdaptiveLockImpl; +import org.apache.rocketmq.store.lock.AdaptiveBackOffLock; +import org.apache.rocketmq.store.lock.AdaptiveBackOffLockImpl; +import org.apache.rocketmq.store.lock.CollisionRetreatLock; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -94,7 +95,7 @@ public class CommitLog implements Swappable { private volatile long beginTimeInLock = 0; - protected final AdaptiveLock putMessageLock; + protected final AdaptiveBackOffLock putMessageLock; protected final TopicQueueLock topicQueueLock; @@ -131,7 +132,8 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = new AdaptiveLockImpl(); + this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? new AdaptiveBackOffLockImpl() : + messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new CollisionRetreatLock(); this.flushDiskWatcher = new FlushDiskWatcher(); @@ -949,6 +951,9 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } + this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock(), + this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()); + int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(msg); @@ -1114,6 +1119,9 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } + this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock(), + this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()); + int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(messageExtBatch); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java index 88d8124c5e3..9fc90768078 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java @@ -16,14 +16,14 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.lock.AdaptiveLock; +import org.apache.rocketmq.store.lock.AdaptiveBackOffLock; import java.util.concurrent.locks.ReentrantLock; /** * Exclusive lock implementation to put message */ -public class PutMessageReentrantLock implements PutMessageLock, AdaptiveLock { +public class PutMessageReentrantLock implements PutMessageLock, AdaptiveBackOffLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 443c238db3e..60efaf62431 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -430,10 +430,10 @@ public class MessageStoreConfig { */ private int spinLockCollisionRetreatOptimalDegree = 1000; - /* - *Critical TPS that triggers the adaptive locking mechanism switch - */ - private int tpsSwapCriticalPoint = 50000; + /** + * Use AdaptiveBackOffLock + **/ + private boolean isUseABSLock = true; public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; @@ -1873,11 +1873,11 @@ public void setSpinLockCollisionRetreatOptimalDegree(int optimalDegree) { this.spinLockCollisionRetreatOptimalDegree = optimalDegree; } - public int getTpsSwapCriticalPoint() { - return tpsSwapCriticalPoint; + public void setUseABSLock(boolean useABSLock) { + this.isUseABSLock = useABSLock; } - public void setTpsSwapCriticalPoint(int tpsSwapCriticalPoint) { - this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; + public boolean getUseABSLock() { + return isUseABSLock; } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java similarity index 90% rename from store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java rename to store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java index 5ae0bbe2ef6..b01c0b805c4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java @@ -18,7 +18,7 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; -public interface AdaptiveLock { +public interface AdaptiveBackOffLock { void lock(); @@ -29,4 +29,7 @@ default void update(MessageStoreConfig messageStoreConfig) { default void swap() { } + + default void isOpen(boolean open, boolean isUseReentrantLock) { + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java similarity index 78% rename from store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java rename to store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index 2a5cff2870e..17f4b61fae3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -27,13 +27,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class AdaptiveLockImpl implements AdaptiveLock { - private AdaptiveLock adaptiveLock; +public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { + private AdaptiveBackOffLock adaptiveLock; //state private AtomicBoolean state = new AtomicBoolean(true); - private Map locks; + private Map locks; private final List tpsTable; @@ -41,7 +41,9 @@ public class AdaptiveLockImpl implements AdaptiveLock { private AtomicInteger currentThreadNum = new AtomicInteger(0); - public AdaptiveLockImpl() { + private AtomicBoolean isOpen = new AtomicBoolean(true); + + public AdaptiveBackOffLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); this.locks.put("Collision", new CollisionRetreatLock()); @@ -69,7 +71,9 @@ public void lock() { public void unlock() { this.adaptiveLock.unlock(); currentThreadNum.decrementAndGet(); - swap(); + if (isOpen.get()) { + swap(); + } } @Override @@ -133,11 +137,34 @@ public void swap() { } } - public List getLocks() { - return (List) this.locks.values(); + @Override + public void isOpen(boolean open, boolean isUseReentrantLock) { + if (open != isOpen.get()) { + isOpen.set(open); + boolean success = false; + do { + success = this.state.compareAndSet(true, false); + } while (!success); + + int currentThreadNum; + do { + currentThreadNum = this.currentThreadNum.get(); + } while (currentThreadNum != 0); + + if (open) { + adaptiveLock = this.locks.get("Collision"); + } else { + adaptiveLock = !isUseReentrantLock ? this.locks.get("Collision") : this.locks.get("Reentrant"); + } + state.set(true); + } + } + + public List getLocks() { + return (List) this.locks.values(); } - public void setLocks(Map locks) { + public void setLocks(Map locks) { this.locks = locks; } @@ -149,7 +176,7 @@ public void setState(boolean state) { this.state.set(state); } - public AdaptiveLock getAdaptiveLock() { + public AdaptiveBackOffLock getAdaptiveLock() { return adaptiveLock; } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java index b7c1cfc64d0..a97807658fc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java @@ -24,7 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class CollisionRetreatLock implements AdaptiveLock { +public class CollisionRetreatLock implements AdaptiveBackOffLock { private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index f174c1c5c7c..4fcb7cc2554 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -25,11 +25,11 @@ public class AdaptiveLockTest { - AdaptiveLockImpl adaptiveLock; + AdaptiveBackOffLockImpl adaptiveLock; @Before public void init() { - adaptiveLock = new AdaptiveLockImpl(); + adaptiveLock = new AdaptiveBackOffLockImpl(); } @Test From c8b4339d6ed5bc315cfe8044471b4dd70db36a51 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 15:54:27 +0800 Subject: [PATCH 18/57] feat:Supports the hot activation of ABS locks --- .../rocketmq/store/lock/AdaptiveBackOffLockImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index 17f4b61fae3..e248263f876 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -191,4 +191,12 @@ public void setTpsSwapCriticalPoint(int tpsSwapCriticalPoint) { public int getTpsSwapCriticalPoint() { return tpsSwapCriticalPoint; } + + public boolean isOpen() { + return this.isOpen.get(); + } + + public void setOpen(boolean open) { + this.isOpen.set(open); + } } From 154fe75a8020326ada16664c98a8cd6c9df0cb3e Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 16:00:04 +0800 Subject: [PATCH 19/57] feat:Supports the hot activation of ABS locks --- .../store/config/MessageStoreConfig.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 60efaf62431..5eeeba9c564 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -424,17 +424,6 @@ public class MessageStoreConfig { private boolean putConsumeQueueDataByFileChannel = true; - /** - * Spin number in the retreat strategy of spin lock - * Default is 1000 - */ - private int spinLockCollisionRetreatOptimalDegree = 1000; - - /** - * Use AdaptiveBackOffLock - **/ - private boolean isUseABSLock = true; - public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } @@ -1864,20 +1853,4 @@ public boolean isTransferMetadataJsonToRocksdb() { public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocksdb) { this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; } - - public int getSpinLockCollisionRetreatOptimalDegree() { - return spinLockCollisionRetreatOptimalDegree; - } - - public void setSpinLockCollisionRetreatOptimalDegree(int optimalDegree) { - this.spinLockCollisionRetreatOptimalDegree = optimalDegree; - } - - public void setUseABSLock(boolean useABSLock) { - this.isUseABSLock = useABSLock; - } - - public boolean getUseABSLock() { - return isUseABSLock; - } } From 568f412731db2ba06c3c868f9cee9f21e956fb02 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 16:02:55 +0800 Subject: [PATCH 20/57] feat:Supports the hot activation of ABS locks --- .../store/config/MessageStoreConfig.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index d8d4cbc126e..95ddaf8b7f1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -430,6 +430,16 @@ public class MessageStoreConfig { private int batchWriteKvCqSize = 16; + /** + * Spin number in the retreat strategy of spin lock + * Default is 1000 + */ + private int spinLockCollisionRetreatOptimalDegree = 1000; + + /** + * Use AdaptiveBackOffLock + **/ + private boolean isUseABSLock = true; public int getBatchWriteKvCqSize() { return batchWriteKvCqSize; @@ -1884,4 +1894,20 @@ public boolean isTransferMetadataJsonToRocksdb() { public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocksdb) { this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; } + + public int getSpinLockCollisionRetreatOptimalDegree() { + return spinLockCollisionRetreatOptimalDegree; + } + + public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) { + this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree; + } + + public boolean getUseABSLock() { + return isUseABSLock; + } + + public void setUseABSLock(boolean useABSLock) { + this.isUseABSLock = useABSLock; + } } From f674f1b7385f6d9cf4e174f5fdc87ffdec974190 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 19:49:46 +0800 Subject: [PATCH 21/57] Optimize code style --- .../org/apache/rocketmq/store/CommitLog.java | 4 +-- .../store/lock/AdaptiveBackOffLockImpl.java | 28 ++++++++------- ...nRetreatLock.java => BackOffSpinLock.java} | 35 +++++++------------ .../rocketmq/store/lock/AdaptiveLockTest.java | 6 ++-- 4 files changed, 34 insertions(+), 39 deletions(-) rename store/src/main/java/org/apache/rocketmq/store/lock/{CollisionRetreatLock.java => BackOffSpinLock.java} (75%) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 5973e28838d..d4c8b56eac4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -62,7 +62,7 @@ import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.lock.AdaptiveBackOffLock; import org.apache.rocketmq.store.lock.AdaptiveBackOffLockImpl; -import org.apache.rocketmq.store.lock.CollisionRetreatLock; +import org.apache.rocketmq.store.lock.BackOffSpinLock; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; @@ -134,7 +134,7 @@ protected PutMessageThreadLocal initialValue() { } }; this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? new AdaptiveBackOffLockImpl() : - messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new CollisionRetreatLock(); + messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new BackOffSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index e248263f876..af736cc8b3d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -33,6 +33,10 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { //state private AtomicBoolean state = new AtomicBoolean(true); + private final static float SWAP_MUTEX_LOCK_RATIO = 0.8f; + + private final static float SPIN_LOCK_ADAPTIVE_RATIO = 1.5f; + private Map locks; private final List tpsTable; @@ -46,18 +50,18 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { public AdaptiveBackOffLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); - this.locks.put("Collision", new CollisionRetreatLock()); + this.locks.put("BackOff", new BackOffSpinLock()); this.tpsTable = new ArrayList<>(2); this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); - adaptiveLock = this.locks.get("Collision"); + adaptiveLock = this.locks.get("BackOff"); } @Override public void lock() { - tpsTable.get(LocalTime.now().getSecond() % 2).getAndIncrement(); + this.tpsTable.get(LocalTime.now().getSecond() % 2).getAndIncrement(); boolean state; do { state = this.state.get(); @@ -94,8 +98,8 @@ public void swap() { return; } - if (this.adaptiveLock instanceof CollisionRetreatLock) { - CollisionRetreatLock lock = (CollisionRetreatLock) this.adaptiveLock; + if (this.adaptiveLock instanceof BackOffSpinLock) { + BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock; int base = Math.min(200 + tps / 200, 500); if (lock.getNumberOfRetreat(slot) * base >= tps) { if (lock.isAdapt()) { @@ -104,12 +108,12 @@ public void swap() { this.tpsSwapCriticalPoint = tps; needSwap = true; } - } else if (lock.getNumberOfRetreat(slot) * base * 3 / 2 <= tps) { + } else if (lock.getNumberOfRetreat(slot) * base * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { lock.adapt(false); } lock.setNumberOfRetreat(slot, 0); } else { - if (tps <= this.tpsSwapCriticalPoint * 4 / 5) { + if (tps <= this.tpsSwapCriticalPoint * SWAP_MUTEX_LOCK_RATIO) { needSwap = true; } } @@ -122,11 +126,11 @@ public void swap() { } while (currentThreadNum != 0); try { - if (this.adaptiveLock instanceof CollisionRetreatLock) { + if (this.adaptiveLock instanceof BackOffSpinLock) { this.adaptiveLock = this.locks.get("Reentrant"); } else { - this.adaptiveLock = this.locks.get("Collision"); - ((CollisionRetreatLock) this.adaptiveLock).adapt(false); + this.adaptiveLock = this.locks.get("BackOff"); + ((BackOffSpinLock) this.adaptiveLock).adapt(false); } } catch (Exception e) { //ignore @@ -152,9 +156,9 @@ public void isOpen(boolean open, boolean isUseReentrantLock) { } while (currentThreadNum != 0); if (open) { - adaptiveLock = this.locks.get("Collision"); + adaptiveLock = this.locks.get("BackOff"); } else { - adaptiveLock = !isUseReentrantLock ? this.locks.get("Collision") : this.locks.get("Reentrant"); + adaptiveLock = !isUseReentrantLock ? this.locks.get("BackOff") : this.locks.get("Reentrant"); } state.set(true); } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java similarity index 75% rename from store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java rename to store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index a97807658fc..b0a532b45a2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/CollisionRetreatLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -21,25 +21,24 @@ import java.time.LocalTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class CollisionRetreatLock implements AdaptiveBackOffLock { +public class BackOffSpinLock implements AdaptiveBackOffLock { private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); private int optimalDegree; - private int initOptimalDegree; + private final static int INITIAL_DEGREE = 1000; - private int maxOptimalDegree; + private final static int MAX_OPTIMAL_DEGREE = 10000; private final List numberOfRetreat; - public CollisionRetreatLock() { - this.initOptimalDegree = 1000; - this.maxOptimalDegree = 10000; - this.optimalDegree = initOptimalDegree; + public BackOffSpinLock() { + this.optimalDegree = INITIAL_DEGREE; numberOfRetreat = new ArrayList<>(2); numberOfRetreat.add(new AtomicInteger(0)); @@ -78,33 +77,25 @@ public void setOptimalDegree(int optimalDegree) { this.optimalDegree = optimalDegree; } - public int getMaxOptimalDegree() { - return maxOptimalDegree; - } - - public void setMaxOptimalDegree(int maxOptimalDegree) { - this.maxOptimalDegree = maxOptimalDegree; - } - public boolean isAdapt() { - return optimalDegree < maxOptimalDegree; + return optimalDegree < MAX_OPTIMAL_DEGREE; } public synchronized void adapt(boolean isRise) { if (isRise) { - if (optimalDegree * 2 <= maxOptimalDegree) { + if (optimalDegree * 2 <= MAX_OPTIMAL_DEGREE) { optimalDegree *= 2; } else { - if (optimalDegree + initOptimalDegree <= maxOptimalDegree) { - optimalDegree += initOptimalDegree; + if (optimalDegree + INITIAL_DEGREE <= MAX_OPTIMAL_DEGREE) { + optimalDegree += INITIAL_DEGREE; } } } else { - if (optimalDegree / 2 >= initOptimalDegree) { + if (optimalDegree / 2 >= INITIAL_DEGREE) { optimalDegree /= 2; } else { - if (optimalDegree > 2 * initOptimalDegree) { - optimalDegree -= initOptimalDegree; + if (optimalDegree > 2 * INITIAL_DEGREE) { + optimalDegree -= MAX_OPTIMAL_DEGREE; } } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 4fcb7cc2554..a0a47e3aece 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -34,7 +34,7 @@ public void init() { @Test public void testAdaptiveLock() throws InterruptedException { - assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); adaptiveLock.lock(); new Thread(new Runnable() { @@ -46,7 +46,7 @@ public void run() { }).start(); Thread.sleep(1000); adaptiveLock.unlock(); - assertEquals(2000, ((CollisionRetreatLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); + assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); @@ -68,6 +68,6 @@ public void run() { adaptiveLock.lock(); Thread.sleep(1000); adaptiveLock.unlock(); - assertTrue(adaptiveLock.getAdaptiveLock() instanceof CollisionRetreatLock); + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); } } From 61a9354dab7df4d32ae5b11f0b8b0d95df9768f0 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 26 Sep 2024 19:56:44 +0800 Subject: [PATCH 22/57] Optimize code style --- .../java/org/apache/rocketmq/store/lock/BackOffSpinLock.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index b0a532b45a2..3655ca89a69 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -21,7 +21,6 @@ import java.time.LocalTime; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; From c7abdd38cbb12fa53700944e3ff04cbbcfed542c Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 27 Sep 2024 15:08:52 +0800 Subject: [PATCH 23/57] Optimize code style --- .../rocketmq/store/config/MessageStoreConfig.java | 2 +- .../store/lock/AdaptiveBackOffLockImpl.java | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 95ddaf8b7f1..870bdb11180 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -439,7 +439,7 @@ public class MessageStoreConfig { /** * Use AdaptiveBackOffLock **/ - private boolean isUseABSLock = true; + private boolean isUseABSLock = false; public int getBatchWriteKvCqSize() { return batchWriteKvCqSize; diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index af736cc8b3d..063924a1e83 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -33,10 +33,12 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { //state private AtomicBoolean state = new AtomicBoolean(true); - private final static float SWAP_MUTEX_LOCK_RATIO = 0.8f; + private final static float SWAP_SPIN_LOCK_RATIO = 0.8f; private final static float SPIN_LOCK_ADAPTIVE_RATIO = 1.5f; + private final static int BASE_SWAP_LOCK_RATIO = 200; + private Map locks; private final List tpsTable; @@ -91,7 +93,7 @@ public void swap() { return; } boolean needSwap = false; - int slot = LocalTime.now().getSecond() % 2 - 1 >= 0 ? 0 : 1; + int slot = 1 - LocalTime.now().getSecond() % 2; int tps = this.tpsTable.get(slot).get() + 1; this.tpsTable.get(slot).set(-1); if (tps == 0) { @@ -100,20 +102,19 @@ public void swap() { if (this.adaptiveLock instanceof BackOffSpinLock) { BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock; - int base = Math.min(200 + tps / 200, 500); - if (lock.getNumberOfRetreat(slot) * base >= tps) { + if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO >= tps) { if (lock.isAdapt()) { lock.adapt(true); } else { this.tpsSwapCriticalPoint = tps; needSwap = true; } - } else if (lock.getNumberOfRetreat(slot) * base * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { + } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { lock.adapt(false); } lock.setNumberOfRetreat(slot, 0); } else { - if (tps <= this.tpsSwapCriticalPoint * SWAP_MUTEX_LOCK_RATIO) { + if (tps <= this.tpsSwapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { needSwap = true; } } From 8de385ef025249450ba2f7d091543dfeb50eaedb Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 27 Sep 2024 17:15:33 +0800 Subject: [PATCH 24/57] Optimize code style --- .../org/apache/rocketmq/store/lock/BackOffSpinLock.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index 3655ca89a69..0c836283871 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -90,12 +90,8 @@ public synchronized void adapt(boolean isRise) { } } } else { - if (optimalDegree / 2 >= INITIAL_DEGREE) { - optimalDegree /= 2; - } else { - if (optimalDegree > 2 * INITIAL_DEGREE) { - optimalDegree -= MAX_OPTIMAL_DEGREE; - } + if (optimalDegree >= 2 * INITIAL_DEGREE) { + optimalDegree -= INITIAL_DEGREE; } } } From 70783b67e2617a8b0af759f7ead47ffe23935e9f Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 27 Sep 2024 17:26:54 +0800 Subject: [PATCH 25/57] Optimize code style --- .../rocketmq/store/lock/AdaptiveBackOffLockImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index 063924a1e83..8b65c2e6957 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -16,6 +16,9 @@ */ package org.apache.rocketmq.store.lock; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.PutMessageReentrantLock; import org.apache.rocketmq.store.config.MessageStoreConfig; @@ -35,9 +38,9 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { private final static float SWAP_SPIN_LOCK_RATIO = 0.8f; - private final static float SPIN_LOCK_ADAPTIVE_RATIO = 1.5f; + private final static int SPIN_LOCK_ADAPTIVE_RATIO = 4; - private final static int BASE_SWAP_LOCK_RATIO = 200; + private final static int BASE_SWAP_LOCK_RATIO = 320; private Map locks; From 56606d33227e1b44efbdb134b190c474e44c1f75 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 27 Sep 2024 17:29:04 +0800 Subject: [PATCH 26/57] Optimize code style --- .../apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java index 8b65c2e6957..bb635275b0f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java @@ -16,9 +16,6 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.PutMessageReentrantLock; import org.apache.rocketmq.store.config.MessageStoreConfig; From 993bcde633ac7780ef05955d5b3057693ee7c565 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 10:26:04 +0800 Subject: [PATCH 27/57] Updated the locking mechanism name --- .../org/apache/rocketmq/store/CommitLog.java | 8 ++++---- .../rocketmq/store/PutMessageReentrantLock.java | 4 ++-- ...OffLock.java => AdaptiveBackOffSpinLock.java} | 2 +- ...mpl.java => AdaptiveBackOffSpinLockImpl.java} | 16 ++++++++-------- .../rocketmq/store/lock/BackOffSpinLock.java | 2 +- .../rocketmq/store/lock/AdaptiveLockTest.java | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) rename store/src/main/java/org/apache/rocketmq/store/lock/{AdaptiveBackOffLock.java => AdaptiveBackOffSpinLock.java} (96%) rename store/src/main/java/org/apache/rocketmq/store/lock/{AdaptiveBackOffLockImpl.java => AdaptiveBackOffSpinLockImpl.java} (92%) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index d4c8b56eac4..db1720ea7d1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,8 +60,8 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; -import org.apache.rocketmq.store.lock.AdaptiveBackOffLock; -import org.apache.rocketmq.store.lock.AdaptiveBackOffLockImpl; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; import org.apache.rocketmq.store.lock.BackOffSpinLock; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; @@ -96,7 +96,7 @@ public class CommitLog implements Swappable { private volatile long beginTimeInLock = 0; - protected final AdaptiveBackOffLock putMessageLock; + protected final AdaptiveBackOffSpinLock putMessageLock; protected final TopicQueueLock topicQueueLock; @@ -133,7 +133,7 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? new AdaptiveBackOffLockImpl() : + this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? new AdaptiveBackOffSpinLockImpl() : messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new BackOffSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java index 9fc90768078..67bd02f532a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java @@ -16,14 +16,14 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.lock.AdaptiveBackOffLock; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; import java.util.concurrent.locks.ReentrantLock; /** * Exclusive lock implementation to put message */ -public class PutMessageReentrantLock implements PutMessageLock, AdaptiveBackOffLock { +public class PutMessageReentrantLock implements PutMessageLock, AdaptiveBackOffSpinLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java similarity index 96% rename from store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java rename to store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index b01c0b805c4..02c946825cb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -18,7 +18,7 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; -public interface AdaptiveBackOffLock { +public interface AdaptiveBackOffSpinLock { void lock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java similarity index 92% rename from store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java rename to store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index bb635275b0f..5965c7e86eb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { - private AdaptiveBackOffLock adaptiveLock; +public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { + private AdaptiveBackOffSpinLock adaptiveLock; //state private AtomicBoolean state = new AtomicBoolean(true); @@ -39,7 +39,7 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { private final static int BASE_SWAP_LOCK_RATIO = 320; - private Map locks; + private Map locks; private final List tpsTable; @@ -49,7 +49,7 @@ public class AdaptiveBackOffLockImpl implements AdaptiveBackOffLock { private AtomicBoolean isOpen = new AtomicBoolean(true); - public AdaptiveBackOffLockImpl() { + public AdaptiveBackOffSpinLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new PutMessageReentrantLock()); this.locks.put("BackOff", new BackOffSpinLock()); @@ -165,11 +165,11 @@ public void isOpen(boolean open, boolean isUseReentrantLock) { } } - public List getLocks() { - return (List) this.locks.values(); + public List getLocks() { + return (List) this.locks.values(); } - public void setLocks(Map locks) { + public void setLocks(Map locks) { this.locks = locks; } @@ -181,7 +181,7 @@ public void setState(boolean state) { this.state.set(state); } - public AdaptiveBackOffLock getAdaptiveLock() { + public AdaptiveBackOffSpinLock getAdaptiveLock() { return adaptiveLock; } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index 0c836283871..a6c2a2a67f4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -24,7 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class BackOffSpinLock implements AdaptiveBackOffLock { +public class BackOffSpinLock implements AdaptiveBackOffSpinLock { private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index a0a47e3aece..b00f57a38c2 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -25,11 +25,11 @@ public class AdaptiveLockTest { - AdaptiveBackOffLockImpl adaptiveLock; + AdaptiveBackOffSpinLockImpl adaptiveLock; @Before public void init() { - adaptiveLock = new AdaptiveBackOffLockImpl(); + adaptiveLock = new AdaptiveBackOffSpinLockImpl(); } @Test From e44da59d479b62377c514b3e235092990310810b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 11:45:15 +0800 Subject: [PATCH 28/57] Optimize the logic of switching to spin locks --- .../store/lock/AdaptiveBackOffSpinLockImpl.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 5965c7e86eb..ee6893fd835 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -29,7 +29,6 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private AdaptiveBackOffSpinLock adaptiveLock; - //state private AtomicBoolean state = new AtomicBoolean(true); @@ -43,7 +42,7 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final List tpsTable; - private int tpsSwapCriticalPoint; + private int swapCriticalPoint; private AtomicInteger currentThreadNum = new AtomicInteger(0); @@ -106,7 +105,7 @@ public void swap() { if (lock.isAdapt()) { lock.adapt(true); } else { - this.tpsSwapCriticalPoint = tps; + this.swapCriticalPoint = tps * currentThreadNum.get(); needSwap = true; } } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { @@ -114,7 +113,7 @@ public void swap() { } lock.setNumberOfRetreat(slot, 0); } else { - if (tps <= this.tpsSwapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { + if (tps * currentThreadNum.get() <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { needSwap = true; } } @@ -189,12 +188,12 @@ public List getTpsTable() { return tpsTable; } - public void setTpsSwapCriticalPoint(int tpsSwapCriticalPoint) { - this.tpsSwapCriticalPoint = tpsSwapCriticalPoint; + public void setTpsSwapCriticalPoint(int swapCriticalPoint) { + this.swapCriticalPoint = swapCriticalPoint; } public int getTpsSwapCriticalPoint() { - return tpsSwapCriticalPoint; + return swapCriticalPoint; } public boolean isOpen() { From fd4272a6652bfe50704340c7d62f72b0eeaf09ea Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 12:24:31 +0800 Subject: [PATCH 29/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index b00f57a38c2..300436bae31 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -63,11 +63,5 @@ public void run() { adaptiveLock.unlock(); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); - - - adaptiveLock.lock(); - Thread.sleep(1000); - adaptiveLock.unlock(); - assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); } } From efafc71b9c61fd3b564791280e33156d7843dda4 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 14:37:37 +0800 Subject: [PATCH 30/57] Optimize the logic of switching to spin locks --- .../lock/AdaptiveBackOffSpinLockImpl.java | 26 ++++++++++++++++--- .../rocketmq/store/lock/AdaptiveLockTest.java | 5 ++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index ee6893fd835..deb130d99d1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -22,8 +22,10 @@ import java.time.LocalTime; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -42,6 +44,8 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final List tpsTable; + private final List threadTable; + private int swapCriticalPoint; private AtomicInteger currentThreadNum = new AtomicInteger(0); @@ -53,6 +57,10 @@ public AdaptiveBackOffSpinLockImpl() { this.locks.put("Reentrant", new PutMessageReentrantLock()); this.locks.put("BackOff", new BackOffSpinLock()); + this.threadTable = new ArrayList<>(2); + this.threadTable.add(new HashSet<>()); + this.threadTable.add(new HashSet<>()); + this.tpsTable = new ArrayList<>(2); this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); @@ -62,7 +70,9 @@ public AdaptiveBackOffSpinLockImpl() { @Override public void lock() { - this.tpsTable.get(LocalTime.now().getSecond() % 2).getAndIncrement(); + int slot = LocalTime.now().getSecond() % 2; + this.threadTable.get(slot).add(Thread.currentThread()); + this.tpsTable.get(slot).getAndIncrement(); boolean state; do { state = this.state.get(); @@ -94,7 +104,9 @@ public void swap() { boolean needSwap = false; int slot = 1 - LocalTime.now().getSecond() % 2; int tps = this.tpsTable.get(slot).get() + 1; + int threadNum = this.threadTable.get(slot).size(); this.tpsTable.get(slot).set(-1); + this.threadTable.get(slot).clear(); if (tps == 0) { return; } @@ -105,7 +117,7 @@ public void swap() { if (lock.isAdapt()) { lock.adapt(true); } else { - this.swapCriticalPoint = tps * currentThreadNum.get(); + this.swapCriticalPoint = tps * threadNum; needSwap = true; } } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { @@ -113,7 +125,7 @@ public void swap() { } lock.setNumberOfRetreat(slot, 0); } else { - if (tps * currentThreadNum.get() <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { + if (tps * threadNum <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { needSwap = true; } } @@ -203,4 +215,12 @@ public boolean isOpen() { public void setOpen(boolean open) { this.isOpen.set(open); } + + public int getSwapCriticalPoint() { + return swapCriticalPoint; + } + + public void setSwapSpinLockRatio(int swapCriticalPoint) { + this.swapCriticalPoint = swapCriticalPoint; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 300436bae31..79d2fd96691 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -63,5 +63,10 @@ public void run() { adaptiveLock.unlock(); } assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); + + adaptiveLock.lock(); + Thread.sleep(1000); + adaptiveLock.unlock(); + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); } } From 293212eba2cdc296f7ba89006c5dc88fdf2869c9 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:36:36 +0800 Subject: [PATCH 31/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/store/CommitLog.java | 16 +++++++------- .../apache/rocketmq/store/PutMessageLock.java | 4 +++- .../store/PutMessageReentrantLock.java | 2 +- .../store/lock/AdaptiveBackOffSpinLock.java | 2 +- .../lock/AdaptiveBackOffSpinLockImpl.java | 21 ++++++++++--------- .../store/lock/BackOffReentrantLock.java | 17 +++++++++++++++ .../rocketmq/store/lock/AdaptiveLockTest.java | 3 ++- 7 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index db1720ea7d1..5d25e1dac3e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -62,7 +62,6 @@ import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; -import org.apache.rocketmq.store.lock.BackOffSpinLock; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; @@ -96,7 +95,9 @@ public class CommitLog implements Swappable { private volatile long beginTimeInLock = 0; - protected final AdaptiveBackOffSpinLock putMessageLock; + protected final PutMessageLock putMessageLock; + + protected final AdaptiveBackOffSpinLock adaptiveBackOffSpinLock; protected final TopicQueueLock topicQueueLock; @@ -133,8 +134,9 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? new AdaptiveBackOffSpinLockImpl() : - messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new BackOffSpinLock(); + this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); + + this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(putMessageLock); this.flushDiskWatcher = new FlushDiskWatcher(); @@ -952,8 +954,7 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock(), - this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()); + this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(msg); @@ -1120,8 +1121,7 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock(), - this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()); + this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(messageExtBatch); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java index 758f4378832..9ca7850a4cb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java @@ -16,10 +16,12 @@ */ package org.apache.rocketmq.store; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; + /** * Used when trying to put message */ -public interface PutMessageLock { +public interface PutMessageLock extends AdaptiveBackOffSpinLock { void lock(); void unlock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java index 67bd02f532a..9e54f6cca72 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java @@ -23,7 +23,7 @@ /** * Exclusive lock implementation to put message */ -public class PutMessageReentrantLock implements PutMessageLock, AdaptiveBackOffSpinLock { +public class PutMessageReentrantLock implements PutMessageLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index 02c946825cb..5317adf34e7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -30,6 +30,6 @@ default void update(MessageStoreConfig messageStoreConfig) { default void swap() { } - default void isOpen(boolean open, boolean isUseReentrantLock) { + default void isOpen(boolean open) { } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index deb130d99d1..8bfe26395f8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -16,7 +16,7 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageReentrantLock; +import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; import java.time.LocalTime; @@ -44,7 +44,7 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final List tpsTable; - private final List threadTable; + private final List> threadTable; private int swapCriticalPoint; @@ -52,10 +52,11 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private AtomicBoolean isOpen = new AtomicBoolean(true); - public AdaptiveBackOffSpinLockImpl() { + public AdaptiveBackOffSpinLockImpl(PutMessageLock putMessageLock) { this.locks = new HashMap<>(); - this.locks.put("Reentrant", new PutMessageReentrantLock()); - this.locks.put("BackOff", new BackOffSpinLock()); + this.locks.put("Reentrant", new BackOffReentrantLock()); + this.locks.put("Spin", new BackOffSpinLock()); + this.locks.put("putMessage", putMessageLock); this.threadTable = new ArrayList<>(2); this.threadTable.add(new HashSet<>()); @@ -65,7 +66,7 @@ public AdaptiveBackOffSpinLockImpl() { this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); - adaptiveLock = this.locks.get("BackOff"); + adaptiveLock = this.locks.get("Spin"); } @Override @@ -141,7 +142,7 @@ public void swap() { if (this.adaptiveLock instanceof BackOffSpinLock) { this.adaptiveLock = this.locks.get("Reentrant"); } else { - this.adaptiveLock = this.locks.get("BackOff"); + this.adaptiveLock = this.locks.get("Spin"); ((BackOffSpinLock) this.adaptiveLock).adapt(false); } } catch (Exception e) { @@ -154,7 +155,7 @@ public void swap() { } @Override - public void isOpen(boolean open, boolean isUseReentrantLock) { + public void isOpen(boolean open) { if (open != isOpen.get()) { isOpen.set(open); boolean success = false; @@ -168,9 +169,9 @@ public void isOpen(boolean open, boolean isUseReentrantLock) { } while (currentThreadNum != 0); if (open) { - adaptiveLock = this.locks.get("BackOff"); + adaptiveLock = this.locks.get("Spin"); } else { - adaptiveLock = !isUseReentrantLock ? this.locks.get("BackOff") : this.locks.get("Reentrant"); + adaptiveLock = this.locks.get("putMessage"); } state.set(true); } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java new file mode 100644 index 00000000000..e1039cda73f --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -0,0 +1,17 @@ +package org.apache.rocketmq.store.lock; + +import java.util.concurrent.locks.ReentrantLock; + +public class BackOffReentrantLock implements AdaptiveBackOffSpinLock{ + private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync + + @Override + public void lock() { + putMessageNormalLock.lock(); + } + + @Override + public void unlock() { + putMessageNormalLock.unlock(); + } +} diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 79d2fd96691..967b2f2b377 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.lock; import org.apache.rocketmq.store.PutMessageReentrantLock; +import org.apache.rocketmq.store.PutMessageSpinLock; import org.junit.Before; import org.junit.Test; @@ -29,7 +30,7 @@ public class AdaptiveLockTest { @Before public void init() { - adaptiveLock = new AdaptiveBackOffSpinLockImpl(); + adaptiveLock = new AdaptiveBackOffSpinLockImpl(new PutMessageSpinLock()); } @Test From 454bba613598f6862a20f5d5cb19d3fe82c3cf0f Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:38:59 +0800 Subject: [PATCH 32/57] Optimize the logic of switching to spin locks --- .../store/lock/BackOffReentrantLock.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java index e1039cda73f..6502dbfc194 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.store.lock; import java.util.concurrent.locks.ReentrantLock; From ba0b4d0baca0ed340b9660b931b6b4feeaa2c245 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:41:03 +0800 Subject: [PATCH 33/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/store/lock/BackOffReentrantLock.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java index 6502dbfc194..90e416419bc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -18,7 +18,7 @@ import java.util.concurrent.locks.ReentrantLock; -public class BackOffReentrantLock implements AdaptiveBackOffSpinLock{ +public class BackOffReentrantLock implements AdaptiveBackOffSpinLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override From e83b0a6fd66831a110bdf32e440eccddb453fe53 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:44:10 +0800 Subject: [PATCH 34/57] Optimize the logic of switching to spin locks --- .../java/org/apache/rocketmq/store/PutMessageReentrantLock.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java index 9e54f6cca72..9aa80d86721 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java @@ -16,8 +16,6 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; - import java.util.concurrent.locks.ReentrantLock; /** From f257111b8f162724aa8fc4df13775d9feed5c99e Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:50:12 +0800 Subject: [PATCH 35/57] Optimize the logic of switching to spin locks --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 967b2f2b377..811418e0af2 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -63,7 +63,7 @@ public void run() { Thread.sleep(1000); adaptiveLock.unlock(); } - assertTrue(adaptiveLock.getAdaptiveLock() instanceof PutMessageReentrantLock); + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock); adaptiveLock.lock(); Thread.sleep(1000); From 6b5a40b32f7172a6292377d31e05e701c88d1a33 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 15:53:52 +0800 Subject: [PATCH 36/57] delete unused import --- .../main/java/org/apache/rocketmq/store/PutMessageLock.java | 2 +- .../apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java | 3 ++- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java index 9ca7850a4cb..1ff42931a42 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java @@ -21,7 +21,7 @@ /** * Used when trying to put message */ -public interface PutMessageLock extends AdaptiveBackOffSpinLock { +public interface PutMessageLock { void lock(); void unlock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index 5317adf34e7..d4343c69b42 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -16,9 +16,10 @@ */ package org.apache.rocketmq.store.lock; +import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; -public interface AdaptiveBackOffSpinLock { +public interface AdaptiveBackOffSpinLock extends PutMessageLock { void lock(); diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 811418e0af2..31bd8298911 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageReentrantLock; import org.apache.rocketmq.store.PutMessageSpinLock; import org.junit.Before; import org.junit.Test; From 1d7bac5c2fea0531af01d4c57c843084ba4fea61 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 16:03:25 +0800 Subject: [PATCH 37/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/store/CommitLog.java | 10 ++---- .../apache/rocketmq/store/PutMessageLock.java | 11 +++++- .../store/lock/AdaptiveBackOffSpinLock.java | 36 ------------------- .../lock/AdaptiveBackOffSpinLockImpl.java | 14 ++++---- .../store/lock/BackOffReentrantLock.java | 4 ++- .../rocketmq/store/lock/BackOffSpinLock.java | 3 +- 6 files changed, 25 insertions(+), 53 deletions(-) delete mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 5d25e1dac3e..6b59c38297d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,7 +60,6 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; -import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; @@ -97,8 +96,6 @@ public class CommitLog implements Swappable { protected final PutMessageLock putMessageLock; - protected final AdaptiveBackOffSpinLock adaptiveBackOffSpinLock; - protected final TopicQueueLock topicQueueLock; private volatile Set fullStorePaths = Collections.emptySet(); @@ -134,9 +131,8 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); - this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(putMessageLock); + this.putMessageLock = new AdaptiveBackOffSpinLockImpl(messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock()); this.flushDiskWatcher = new FlushDiskWatcher(); @@ -954,7 +950,7 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); + this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(msg); @@ -1121,7 +1117,7 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); + this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(messageExtBatch); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java index 1ff42931a42..acd34ef2645 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java @@ -16,7 +16,7 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; +import org.apache.rocketmq.store.config.MessageStoreConfig; /** * Used when trying to put message @@ -25,4 +25,13 @@ public interface PutMessageLock { void lock(); void unlock(); + + default void update(MessageStoreConfig messageStoreConfig) { + } + + default void swap() { + } + + default void isOpen(boolean open) { + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java deleted file mode 100644 index d4343c69b42..00000000000 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.store.lock; - -import org.apache.rocketmq.store.PutMessageLock; -import org.apache.rocketmq.store.config.MessageStoreConfig; - -public interface AdaptiveBackOffSpinLock extends PutMessageLock { - - void lock(); - - void unlock(); - - default void update(MessageStoreConfig messageStoreConfig) { - } - - default void swap() { - } - - default void isOpen(boolean open) { - } -} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 8bfe26395f8..59e5328b87f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -29,8 +29,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { - private AdaptiveBackOffSpinLock adaptiveLock; +public class AdaptiveBackOffSpinLockImpl implements PutMessageLock { + private PutMessageLock adaptiveLock; //state private AtomicBoolean state = new AtomicBoolean(true); @@ -40,7 +40,7 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final static int BASE_SWAP_LOCK_RATIO = 320; - private Map locks; + private Map locks; private final List tpsTable; @@ -177,11 +177,11 @@ public void isOpen(boolean open) { } } - public List getLocks() { - return (List) this.locks.values(); + public List getLocks() { + return (List) this.locks.values(); } - public void setLocks(Map locks) { + public void setLocks(Map locks) { this.locks = locks; } @@ -193,7 +193,7 @@ public void setState(boolean state) { this.state.set(state); } - public AdaptiveBackOffSpinLock getAdaptiveLock() { + public PutMessageLock getAdaptiveLock() { return adaptiveLock; } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java index 90e416419bc..1a19b93a4a5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -16,9 +16,11 @@ */ package org.apache.rocketmq.store.lock; +import org.apache.rocketmq.store.PutMessageLock; + import java.util.concurrent.locks.ReentrantLock; -public class BackOffReentrantLock implements AdaptiveBackOffSpinLock { +public class BackOffReentrantLock implements PutMessageLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index a6c2a2a67f4..ba24aa7bed2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store.lock; +import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; import java.time.LocalTime; @@ -24,7 +25,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class BackOffSpinLock implements AdaptiveBackOffSpinLock { +public class BackOffSpinLock implements PutMessageLock { private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); From 2174bd54ddf913d0e3432d65d6522fd4d2915416 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 17:33:43 +0800 Subject: [PATCH 38/57] Revert "Optimize the logic of switching to spin locks" This reverts commit 1d7bac5c2fea0531af01d4c57c843084ba4fea61. --- .../org/apache/rocketmq/store/CommitLog.java | 10 ++++-- .../apache/rocketmq/store/PutMessageLock.java | 11 +----- .../store/lock/AdaptiveBackOffSpinLock.java | 36 +++++++++++++++++++ .../lock/AdaptiveBackOffSpinLockImpl.java | 14 ++++---- .../store/lock/BackOffReentrantLock.java | 4 +-- .../rocketmq/store/lock/BackOffSpinLock.java | 3 +- 6 files changed, 53 insertions(+), 25 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 6b59c38297d..5d25e1dac3e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,6 +60,7 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; @@ -96,6 +97,8 @@ public class CommitLog implements Swappable { protected final PutMessageLock putMessageLock; + protected final AdaptiveBackOffSpinLock adaptiveBackOffSpinLock; + protected final TopicQueueLock topicQueueLock; private volatile Set fullStorePaths = Collections.emptySet(); @@ -131,8 +134,9 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; + this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); - this.putMessageLock = new AdaptiveBackOffSpinLockImpl(messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock()); + this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(putMessageLock); this.flushDiskWatcher = new FlushDiskWatcher(); @@ -950,7 +954,7 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); + this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(msg); @@ -1117,7 +1121,7 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.putMessageLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); + this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(messageExtBatch); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java index acd34ef2645..1ff42931a42 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java @@ -16,7 +16,7 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; /** * Used when trying to put message @@ -25,13 +25,4 @@ public interface PutMessageLock { void lock(); void unlock(); - - default void update(MessageStoreConfig messageStoreConfig) { - } - - default void swap() { - } - - default void isOpen(boolean open) { - } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java new file mode 100644 index 00000000000..d4343c69b42 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.PutMessageLock; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +public interface AdaptiveBackOffSpinLock extends PutMessageLock { + + void lock(); + + void unlock(); + + default void update(MessageStoreConfig messageStoreConfig) { + } + + default void swap() { + } + + default void isOpen(boolean open) { + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 59e5328b87f..8bfe26395f8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -29,8 +29,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class AdaptiveBackOffSpinLockImpl implements PutMessageLock { - private PutMessageLock adaptiveLock; +public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { + private AdaptiveBackOffSpinLock adaptiveLock; //state private AtomicBoolean state = new AtomicBoolean(true); @@ -40,7 +40,7 @@ public class AdaptiveBackOffSpinLockImpl implements PutMessageLock { private final static int BASE_SWAP_LOCK_RATIO = 320; - private Map locks; + private Map locks; private final List tpsTable; @@ -177,11 +177,11 @@ public void isOpen(boolean open) { } } - public List getLocks() { - return (List) this.locks.values(); + public List getLocks() { + return (List) this.locks.values(); } - public void setLocks(Map locks) { + public void setLocks(Map locks) { this.locks = locks; } @@ -193,7 +193,7 @@ public void setState(boolean state) { this.state.set(state); } - public PutMessageLock getAdaptiveLock() { + public AdaptiveBackOffSpinLock getAdaptiveLock() { return adaptiveLock; } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java index 1a19b93a4a5..90e416419bc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -16,11 +16,9 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageLock; - import java.util.concurrent.locks.ReentrantLock; -public class BackOffReentrantLock implements PutMessageLock { +public class BackOffReentrantLock implements AdaptiveBackOffSpinLock { private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index ba24aa7bed2..a6c2a2a67f4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; import java.time.LocalTime; @@ -25,7 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -public class BackOffSpinLock implements PutMessageLock { +public class BackOffSpinLock implements AdaptiveBackOffSpinLock { private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); From b68b059c1837b65bb3ba7a721c5b88be6d8251f2 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 18:07:19 +0800 Subject: [PATCH 39/57] Optimize the logic of switching to spin locks --- .../src/main/java/org/apache/rocketmq/store/CommitLog.java | 6 +----- .../main/java/org/apache/rocketmq/store/PutMessageLock.java | 2 -- .../apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java | 3 +-- .../rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java | 4 +--- .../org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 3 +-- 5 files changed, 4 insertions(+), 14 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 5d25e1dac3e..393d7dc07a4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -136,7 +136,7 @@ protected PutMessageThreadLocal initialValue() { }; this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); - this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(putMessageLock); + this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); this.flushDiskWatcher = new FlushDiskWatcher(); @@ -954,8 +954,6 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); - int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(msg); @@ -1121,8 +1119,6 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); } - this.adaptiveBackOffSpinLock.isOpen(this.defaultMessageStore.getMessageStoreConfig().getUseABSLock()); - int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas(); boolean needHandleHA = needHandleHA(messageExtBatch); diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java index 1ff42931a42..758f4378832 100644 --- a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java @@ -16,8 +16,6 @@ */ package org.apache.rocketmq.store; -import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; - /** * Used when trying to put message */ diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index d4343c69b42..5317adf34e7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -16,10 +16,9 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; -public interface AdaptiveBackOffSpinLock extends PutMessageLock { +public interface AdaptiveBackOffSpinLock { void lock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 8bfe26395f8..14a581fac48 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; import java.time.LocalTime; @@ -52,11 +51,10 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private AtomicBoolean isOpen = new AtomicBoolean(true); - public AdaptiveBackOffSpinLockImpl(PutMessageLock putMessageLock) { + public AdaptiveBackOffSpinLockImpl() { this.locks = new HashMap<>(); this.locks.put("Reentrant", new BackOffReentrantLock()); this.locks.put("Spin", new BackOffSpinLock()); - this.locks.put("putMessage", putMessageLock); this.threadTable = new ArrayList<>(2); this.threadTable.add(new HashSet<>()); diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 31bd8298911..326e4f9ce6c 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store.lock; -import org.apache.rocketmq.store.PutMessageSpinLock; import org.junit.Before; import org.junit.Test; @@ -29,7 +28,7 @@ public class AdaptiveLockTest { @Before public void init() { - adaptiveLock = new AdaptiveBackOffSpinLockImpl(new PutMessageSpinLock()); + adaptiveLock = new AdaptiveBackOffSpinLockImpl(); } @Test From 1b546f29465c4a16d28c8ba96584c8bc0471210a Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 9 Oct 2024 19:25:43 +0800 Subject: [PATCH 40/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/store/CommitLog.java | 4 ++- .../store/lock/AdaptiveBackOffSpinLock.java | 14 +++++++- .../lock/AdaptiveBackOffSpinLockImpl.java | 35 ++----------------- 3 files changed, 18 insertions(+), 35 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 393d7dc07a4..04553008a90 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -134,10 +134,12 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); + this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? this.adaptiveBackOffSpinLock : + messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); + this.flushDiskWatcher = new FlushDiskWatcher(); this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum()); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index 5317adf34e7..b4f999d1dbc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -16,20 +16,32 @@ */ package org.apache.rocketmq.store.lock; +import org.apache.rocketmq.store.PutMessageLock; import org.apache.rocketmq.store.config.MessageStoreConfig; -public interface AdaptiveBackOffSpinLock { +public interface AdaptiveBackOffSpinLock extends PutMessageLock { void lock(); void unlock(); + /** + * Configuration update + * @param messageStoreConfig + */ default void update(MessageStoreConfig messageStoreConfig) { } + /** + * Locking mechanism switching + */ default void swap() { } + /** + * A switch that controls the opening and closing of the locking mechanism + * @param open + */ default void isOpen(boolean open) { } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 14a581fac48..1834466c7fd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -152,29 +152,6 @@ public void swap() { } } - @Override - public void isOpen(boolean open) { - if (open != isOpen.get()) { - isOpen.set(open); - boolean success = false; - do { - success = this.state.compareAndSet(true, false); - } while (!success); - - int currentThreadNum; - do { - currentThreadNum = this.currentThreadNum.get(); - } while (currentThreadNum != 0); - - if (open) { - adaptiveLock = this.locks.get("Spin"); - } else { - adaptiveLock = this.locks.get("putMessage"); - } - state.set(true); - } - } - public List getLocks() { return (List) this.locks.values(); } @@ -199,11 +176,11 @@ public List getTpsTable() { return tpsTable; } - public void setTpsSwapCriticalPoint(int swapCriticalPoint) { + public void setSwapCriticalPoint(int swapCriticalPoint) { this.swapCriticalPoint = swapCriticalPoint; } - public int getTpsSwapCriticalPoint() { + public int getSwapCriticalPoint() { return swapCriticalPoint; } @@ -214,12 +191,4 @@ public boolean isOpen() { public void setOpen(boolean open) { this.isOpen.set(open); } - - public int getSwapCriticalPoint() { - return swapCriticalPoint; - } - - public void setSwapSpinLockRatio(int swapCriticalPoint) { - this.swapCriticalPoint = swapCriticalPoint; - } } From ade9084b882c7e04fd1989cbabc18f7c5170638a Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 11 Oct 2024 14:13:48 +0800 Subject: [PATCH 41/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/common/BrokerConfig.java | 13 +++++++++++++ .../java/org/apache/rocketmq/store/CommitLog.java | 2 +- .../rocketmq/store/config/MessageStoreConfig.java | 13 ------------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 2123e9b339d..f3dba4e577f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -427,6 +427,11 @@ public class BrokerConfig extends BrokerIdentity { // if false, will still rewrite ck after max times 17 private boolean skipWhenCKRePutReachMaxTimes = false; + /** + * Use AdaptiveBackOffLock + **/ + private boolean useABSLock = false; + public String getConfigBlackList() { return configBlackList; } @@ -1859,4 +1864,12 @@ public int getUpdateNameServerAddrPeriod() { public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) { this.updateNameServerAddrPeriod = updateNameServerAddrPeriod; } + + public void setUseABSLock(boolean useABSLock) { + this.useABSLock = useABSLock; + } + + public boolean getUseABSLock() { + return useABSLock; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 04553008a90..b6d56a64248 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -137,7 +137,7 @@ protected PutMessageThreadLocal initialValue() { this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); - this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? this.adaptiveBackOffSpinLock : + this.putMessageLock = messageStore.getBrokerConfig().getUseABSLock() ? this.adaptiveBackOffSpinLock : messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 870bdb11180..14b9c86d0a2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -436,11 +436,6 @@ public class MessageStoreConfig { */ private int spinLockCollisionRetreatOptimalDegree = 1000; - /** - * Use AdaptiveBackOffLock - **/ - private boolean isUseABSLock = false; - public int getBatchWriteKvCqSize() { return batchWriteKvCqSize; } @@ -1902,12 +1897,4 @@ public int getSpinLockCollisionRetreatOptimalDegree() { public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) { this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree; } - - public boolean getUseABSLock() { - return isUseABSLock; - } - - public void setUseABSLock(boolean useABSLock) { - this.isUseABSLock = useABSLock; - } } From 1323ed0be6905a47e3a191df24cfdb03233608a8 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 11 Oct 2024 16:55:57 +0800 Subject: [PATCH 42/57] Optimize the logic of switching to spin locks --- .../rocketmq/store/lock/AdaptiveBackOffSpinLock.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index b4f999d1dbc..d0bec86112e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -37,11 +37,4 @@ default void update(MessageStoreConfig messageStoreConfig) { */ default void swap() { } - - /** - * A switch that controls the opening and closing of the locking mechanism - * @param open - */ - default void isOpen(boolean open) { - } } From 02e63281b8bcd869e7670b2c95ec9ea4746788ad Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 11 Oct 2024 16:57:38 +0800 Subject: [PATCH 43/57] Optimize the logic of switching to spin locks --- .../src/main/java/org/apache/rocketmq/store/CommitLog.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index b6d56a64248..762f36d353a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -97,8 +97,6 @@ public class CommitLog implements Swappable { protected final PutMessageLock putMessageLock; - protected final AdaptiveBackOffSpinLock adaptiveBackOffSpinLock; - protected final TopicQueueLock topicQueueLock; private volatile Set fullStorePaths = Collections.emptySet(); @@ -135,9 +133,9 @@ protected PutMessageThreadLocal initialValue() { } }; - this.adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); + AdaptiveBackOffSpinLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); - this.putMessageLock = messageStore.getBrokerConfig().getUseABSLock() ? this.adaptiveBackOffSpinLock : + this.putMessageLock = messageStore.getBrokerConfig().getUseABSLock() ? adaptiveBackOffSpinLock : messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); From 081b5e086261a379e02c1363fd6b4c8a7b98d710 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 11 Oct 2024 17:45:50 +0800 Subject: [PATCH 44/57] Optimize the logic of switching to spin locks --- .../org/apache/rocketmq/common/BrokerConfig.java | 13 ------------- .../rocketmq/store/config/MessageStoreConfig.java | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index f3dba4e577f..2123e9b339d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -427,11 +427,6 @@ public class BrokerConfig extends BrokerIdentity { // if false, will still rewrite ck after max times 17 private boolean skipWhenCKRePutReachMaxTimes = false; - /** - * Use AdaptiveBackOffLock - **/ - private boolean useABSLock = false; - public String getConfigBlackList() { return configBlackList; } @@ -1864,12 +1859,4 @@ public int getUpdateNameServerAddrPeriod() { public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) { this.updateNameServerAddrPeriod = updateNameServerAddrPeriod; } - - public void setUseABSLock(boolean useABSLock) { - this.useABSLock = useABSLock; - } - - public boolean getUseABSLock() { - return useABSLock; - } } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 14b9c86d0a2..59d29856fe2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -436,6 +436,11 @@ public class MessageStoreConfig { */ private int spinLockCollisionRetreatOptimalDegree = 1000; + /** + * Use AdaptiveBackOffLock + **/ + private boolean useABSLock = false; + public int getBatchWriteKvCqSize() { return batchWriteKvCqSize; } @@ -1897,4 +1902,12 @@ public int getSpinLockCollisionRetreatOptimalDegree() { public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) { this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree; } + + public void setUseABSLock(boolean useABSLock) { + this.useABSLock = useABSLock; + } + + public boolean getUseABSLock() { + return useABSLock; + } } From 882f7edbe875cc34b678c03e7ee6b56267187cb3 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Fri, 11 Oct 2024 17:51:39 +0800 Subject: [PATCH 45/57] Optimize the logic of switching to spin locks --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 762f36d353a..17fbe06af44 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -135,7 +135,7 @@ protected PutMessageThreadLocal initialValue() { AdaptiveBackOffSpinLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); - this.putMessageLock = messageStore.getBrokerConfig().getUseABSLock() ? adaptiveBackOffSpinLock : + this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? adaptiveBackOffSpinLock : messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); From cb96e72f92342ee30218df0fb3751da4b03e715d Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 15:02:03 +0800 Subject: [PATCH 46/57] Optimized locking logic --- .../org/apache/rocketmq/store/lock/BackOffSpinLock.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java index a6c2a2a67f4..f754970a054 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -54,7 +54,11 @@ public void lock() { } } numberOfRetreat.get(LocalTime.now().getSecond() % 2).getAndIncrement(); - Thread.yield(); + try { + Thread.sleep(0); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } From 7b4401e459582bafadcd0160012815b4ba9bca09 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 19:22:08 +0800 Subject: [PATCH 47/57] Optimized locking logic --- .../store/lock/AdaptiveBackOffSpinLockImpl.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 1834466c7fd..e582ee8eb1e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -21,10 +21,9 @@ import java.time.LocalTime; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -43,7 +42,7 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final List tpsTable; - private final List> threadTable; + private final List> threadTable; private int swapCriticalPoint; @@ -57,8 +56,8 @@ public AdaptiveBackOffSpinLockImpl() { this.locks.put("Spin", new BackOffSpinLock()); this.threadTable = new ArrayList<>(2); - this.threadTable.add(new HashSet<>()); - this.threadTable.add(new HashSet<>()); + this.threadTable.add(new ConcurrentHashMap<>()); + this.threadTable.add(new ConcurrentHashMap<>()); this.tpsTable = new ArrayList<>(2); this.tpsTable.add(new AtomicInteger(0)); @@ -70,7 +69,7 @@ public AdaptiveBackOffSpinLockImpl() { @Override public void lock() { int slot = LocalTime.now().getSecond() % 2; - this.threadTable.get(slot).add(Thread.currentThread()); + this.threadTable.get(slot).put(Thread.currentThread(), null); this.tpsTable.get(slot).getAndIncrement(); boolean state; do { From 3ed1e3594bdd3f6a49e61975d86d9cc2118db7d2 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 20:43:00 +0800 Subject: [PATCH 48/57] Optimized locking logic --- .../rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index e582ee8eb1e..c75c0b7220f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -42,7 +42,7 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { private final List tpsTable; - private final List> threadTable; + private final List> threadTable; private int swapCriticalPoint; @@ -69,7 +69,7 @@ public AdaptiveBackOffSpinLockImpl() { @Override public void lock() { int slot = LocalTime.now().getSecond() % 2; - this.threadTable.get(slot).put(Thread.currentThread(), null); + this.threadTable.get(slot).put(Thread.currentThread(), Byte.MAX_VALUE); this.tpsTable.get(slot).getAndIncrement(); boolean state; do { From c401dcab422d6ce13185667b8c26dccdb981329b Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 22:25:56 +0800 Subject: [PATCH 49/57] fix test --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 326e4f9ce6c..5f52dd875d9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -46,9 +46,7 @@ public void run() { Thread.sleep(1000); adaptiveLock.unlock(); assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - - - + for (int i = 0; i <= 8; i++) { adaptiveLock.lock(); new Thread(new Runnable() { From 3c6bcef5af2bb22ae77f21d91640102c230a38d1 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 22:45:40 +0800 Subject: [PATCH 50/57] fix test --- .../apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index c75c0b7220f..2c753ce6d4d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -69,7 +69,7 @@ public AdaptiveBackOffSpinLockImpl() { @Override public void lock() { int slot = LocalTime.now().getSecond() % 2; - this.threadTable.get(slot).put(Thread.currentThread(), Byte.MAX_VALUE); + this.threadTable.get(slot).putIfAbsent(Thread.currentThread(), Byte.MAX_VALUE); this.tpsTable.get(slot).getAndIncrement(); boolean state; do { From 0302a49ef19318354dd92b7f5f1509d6c5811f5c Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 16 Oct 2024 23:21:35 +0800 Subject: [PATCH 51/57] fix test --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 5f52dd875d9..301e2141241 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -47,7 +47,7 @@ public void run() { adaptiveLock.unlock(); assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - for (int i = 0; i <= 8; i++) { + for (int i = 0; i <= 5; i++) { adaptiveLock.lock(); new Thread(new Runnable() { @Override @@ -59,6 +59,7 @@ public void run() { Thread.sleep(1000); adaptiveLock.unlock(); } + assertEquals(4, adaptiveLock.getSwapCriticalPoint()); assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock); adaptiveLock.lock(); From 530cb62a1f9a8ac509b028428e4d1516c8f99250 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 17 Oct 2024 00:28:21 +0800 Subject: [PATCH 52/57] fix test --- .../rocketmq/store/lock/AdaptiveLockTest.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 301e2141241..0abb926c248 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -19,6 +19,8 @@ import org.junit.Before; import org.junit.Test; +import java.util.concurrent.CountDownLatch; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -34,30 +36,45 @@ public void init() { @Test public void testAdaptiveLock() throws InterruptedException { assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); - + CountDownLatch countDownLatch = new CountDownLatch(1); adaptiveLock.lock(); new Thread(new Runnable() { @Override public void run() { adaptiveLock.lock(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //ignore + } adaptiveLock.unlock(); + countDownLatch.countDown(); } }).start(); Thread.sleep(1000); adaptiveLock.unlock(); assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); - + countDownLatch.await(); + for (int i = 0; i <= 5; i++) { + CountDownLatch countDownLatch1 = new CountDownLatch(1); adaptiveLock.lock(); new Thread(new Runnable() { @Override public void run() { adaptiveLock.lock(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //ignore + } adaptiveLock.unlock(); + countDownLatch1.countDown(); } }).start(); Thread.sleep(1000); adaptiveLock.unlock(); + countDownLatch1.await(); } assertEquals(4, adaptiveLock.getSwapCriticalPoint()); assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock); From cfc02400191335782633fceeba60eb4b6089ea7c Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 17 Oct 2024 15:41:45 +0800 Subject: [PATCH 53/57] Optimize code style --- .../org/apache/rocketmq/store/CommitLog.java | 2 +- .../store/lock/AdaptiveBackOffSpinLock.java | 5 ----- .../lock/AdaptiveBackOffSpinLockImpl.java | 22 +++++++++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 17fbe06af44..61d592b4c85 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -133,7 +133,7 @@ protected PutMessageThreadLocal initialValue() { } }; - AdaptiveBackOffSpinLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); + PutMessageLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? adaptiveBackOffSpinLock : messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java index d0bec86112e..96200bcc15b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -20,11 +20,6 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; public interface AdaptiveBackOffSpinLock extends PutMessageLock { - - void lock(); - - void unlock(); - /** * Configuration update * @param messageStoreConfig diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 2c753ce6d4d..9ba00c9967e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -32,12 +32,21 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { //state private AtomicBoolean state = new AtomicBoolean(true); + // Used to determine the switchover between a mutex lock and a spin lock private final static float SWAP_SPIN_LOCK_RATIO = 0.8f; + // It is used to adjust the spin number K of the escape spin lock + // When (retreat number / TPS) <= (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO), K is decreased private final static int SPIN_LOCK_ADAPTIVE_RATIO = 4; + // It is used to adjust the spin number K of the escape spin lock + // When (retreat number / TPS) >= (1 / BASE_SWAP_ADAPTIVE_RATIO), K is increased private final static int BASE_SWAP_LOCK_RATIO = 320; + private final static String BACK_OFF_SPIN_LOCK = "SpinLock"; + + private final static String REENTRANT_LOCK = "ReentrantLock"; + private Map locks; private final List tpsTable; @@ -52,8 +61,8 @@ public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { public AdaptiveBackOffSpinLockImpl() { this.locks = new HashMap<>(); - this.locks.put("Reentrant", new BackOffReentrantLock()); - this.locks.put("Spin", new BackOffSpinLock()); + this.locks.put(REENTRANT_LOCK, new BackOffReentrantLock()); + this.locks.put(BACK_OFF_SPIN_LOCK, new BackOffSpinLock()); this.threadTable = new ArrayList<>(2); this.threadTable.add(new ConcurrentHashMap<>()); @@ -111,10 +120,14 @@ public void swap() { if (this.adaptiveLock instanceof BackOffSpinLock) { BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock; + // Avoid frequent adjustment of K, and make a reasonable range through experiments + // reasonable range : (retreat number / TPS) > (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO) && + // (retreat number / TPS) < (1 / BASE_SWAP_ADAPTIVE_RATIO) if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO >= tps) { if (lock.isAdapt()) { lock.adapt(true); } else { + // It is used to switch between mutex lock and spin lock this.swapCriticalPoint = tps * threadNum; needSwap = true; } @@ -130,6 +143,7 @@ public void swap() { if (needSwap) { if (this.state.compareAndSet(true, false)) { + // Ensures that no threads are in contention locks as well as in critical zones int currentThreadNum; do { currentThreadNum = this.currentThreadNum.get(); @@ -137,9 +151,9 @@ public void swap() { try { if (this.adaptiveLock instanceof BackOffSpinLock) { - this.adaptiveLock = this.locks.get("Reentrant"); + this.adaptiveLock = this.locks.get(REENTRANT_LOCK); } else { - this.adaptiveLock = this.locks.get("Spin"); + this.adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK); ((BackOffSpinLock) this.adaptiveLock).adapt(false); } } catch (Exception e) { From f5260561728b748463b24c294b41cdf4c47da884 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 17 Oct 2024 15:42:32 +0800 Subject: [PATCH 54/57] Optimize code style --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 61d592b4c85..9fb8c411f09 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,7 +60,6 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; -import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLock; import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.MultiDispatchUtils; From 447ad62aacde48f378572d9afa2e5a4b9c3fbfcb Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 17 Oct 2024 15:54:19 +0800 Subject: [PATCH 55/57] fix test --- .../apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java index 9ba00c9967e..b4abb082718 100644 --- a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -72,7 +72,7 @@ public AdaptiveBackOffSpinLockImpl() { this.tpsTable.add(new AtomicInteger(0)); this.tpsTable.add(new AtomicInteger(0)); - adaptiveLock = this.locks.get("Spin"); + adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK); } @Override From e8864475146c39b7f6f812626f02f766e6c064c7 Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Thu, 17 Oct 2024 16:25:47 +0800 Subject: [PATCH 56/57] fix test --- .../java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java index 0abb926c248..ac1b3c60cc0 100644 --- a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -76,7 +76,6 @@ public void run() { adaptiveLock.unlock(); countDownLatch1.await(); } - assertEquals(4, adaptiveLock.getSwapCriticalPoint()); assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock); adaptiveLock.lock(); From 39180a30b289be2311612ad8a26532eb549ce99c Mon Sep 17 00:00:00 2001 From: wanghuaiyuan Date: Wed, 23 Oct 2024 17:15:07 +0800 Subject: [PATCH 57/57] optimize client rebalancing logic --- .../apache/rocketmq/store/config/MessageStoreConfig.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 9b1743eb784..e31c03dd22b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -456,14 +456,6 @@ public class MessageStoreConfig { **/ private boolean useABSLock = false; - public int getBatchWriteKvCqSize() { - return batchWriteKvCqSize; - } - - public void setBatchWriteKvCqSize(int batchWriteKvCqSize) { - this.batchWriteKvCqSize = batchWriteKvCqSize; - } - public boolean isRocksdbCQDoubleWriteEnable() { return rocksdbCQDoubleWriteEnable; }