fix some tests

Signed-off-by: garyschulte <garyschulte@gmail.com>
garyschulte · Jul 14, 2023 · 8645360 · 8645360
1 parent 33fb8ff
commit 8645360
Show file tree

Hide file tree

Showing 19 changed files with 128 additions and 63 deletions.
diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -621,10 +621,7 @@ public BesuController build() {
 
     Optional<Pruner> maybePruner = Optional.empty();
     if (isPruningEnabled) {
-      if (!storageProvider.isWorldStateIterable()) {
-        LOG.warn(
-            "Cannot enable pruning with current database version. Disabling. Resync to get the latest database version or disable pruning explicitly on the command line to remove this warning.");
-      } else if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+      if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
         LOG.warn(
             "Cannot enable pruning with Bonsai data storage format. Disabling. Change the data storage format or disable pruning explicitly on the command line to remove this warning.");
       } else {

diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java
@@ -134,7 +134,6 @@ public void setup() {
     when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST))
         .thenReturn(worldStateStorage);
     when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage);
-    when(storageProvider.isWorldStateIterable()).thenReturn(true);
 
     when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true);
     when(worldStatePreimageStorage.updater())

diff --git a/...integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java b/...integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java
@@ -33,6 +33,7 @@
 import org.hyperledger.besu.ethereum.worldstate.Pruner.PruningPhase;
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
+import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 import org.hyperledger.besu.testutil.MockExecutorService;
 
 import java.util.ArrayList;

diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java
@@ -41,6 +41,4 @@ BlockchainStorage createBlockchainStorage(
   KeyValueStorage getStorageBySegmentIdentifier(SegmentIdentifier segment);
 
   SegmentedKeyValueStorage getStorageBySegmentIdentifiers(List<SegmentIdentifier> segment);
-
-  boolean isWorldStateIterable();
 }
diff --git a/...src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/...src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java
@@ -42,25 +42,19 @@
 public class KeyValueStorageProvider implements StorageProvider {
   private static final Logger LOG = LoggerFactory.getLogger(StorageProvider.class);
 
-  public static final boolean SEGMENT_ISOLATION_SUPPORTED = true;
-  public static final boolean SNAPSHOT_ISOLATION_UNSUPPORTED = false;
-
   protected final Function<List<SegmentIdentifier>, SegmentedKeyValueStorage>
       segmentedStorageCreator;
   private final KeyValueStorage worldStatePreimageStorage;
-  private final boolean isWorldStateIterable;
   protected final Map<List<SegmentIdentifier>, SegmentedKeyValueStorage> storageInstances =
       new HashMap<>();
   private final ObservableMetricsSystem metricsSystem;
 
   public KeyValueStorageProvider(
       final Function<List<SegmentIdentifier>, SegmentedKeyValueStorage> segmentedStorageCreator,
       final KeyValueStorage worldStatePreimageStorage,
-      final boolean segmentIsolationSupported,
       final ObservableMetricsSystem metricsSystem) {
     this.segmentedStorageCreator = segmentedStorageCreator;
     this.worldStatePreimageStorage = worldStatePreimageStorage;
-    this.isWorldStateIterable = segmentIsolationSupported;
     this.metricsSystem = metricsSystem;
   }
 
@@ -106,11 +100,6 @@ public SegmentedKeyValueStorage getStorageBySegmentIdentifiers(
     return segmentedStorageCreator.apply(segments);
   }
 
-  @Override
-  public boolean isWorldStateIterable() {
-    return isWorldStateIterable;
-  }
-
   @Override
   public void close() throws IOException {
     storageInstances.entrySet().stream()

diff --git a/...n/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java b/...n/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java
@@ -23,8 +23,6 @@
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageFactory;
 import org.hyperledger.besu.services.kvstore.LimitedInMemoryKeyValueStorage;
 
-import java.util.List;
-
 public class KeyValueStorageProviderBuilder {
 
   private static final long DEFAULT_WORLD_STATE_PRE_IMAGE_CACHE_SIZE = 5_000L;
@@ -60,14 +58,9 @@ public KeyValueStorageProvider build() {
     final KeyValueStorage worldStatePreImageStorage =
         new LimitedInMemoryKeyValueStorage(DEFAULT_WORLD_STATE_PRE_IMAGE_CACHE_SIZE);
 
-    // this tickles init needed for isSegmentIsolationSupported
-    storageFactory.create(
-        List.of(KeyValueSegmentIdentifier.BLOCKCHAIN), commonConfiguration, metricsSystem);
-
     return new KeyValueStorageProvider(
         segments -> storageFactory.create(segments, commonConfiguration, metricsSystem),
         worldStatePreImageStorage,
-        storageFactory.isSegmentIsolationSupported(),
         (ObservableMetricsSystem) metricsSystem);
   }
 }
diff --git a/...test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/...test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java
@@ -41,7 +41,6 @@ public InMemoryKeyValueStorageProvider() {
     super(
         segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(),
         new InMemoryKeyValueStorage(),
-        SEGMENT_ISOLATION_SUPPORTED,
         new NoOpMetricsSystem());
   }
 

diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java
@@ -17,6 +17,10 @@
 package org.hyperledger.besu.ethereum.bonsai;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
 
 import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
@@ -26,11 +30,15 @@
 import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
 import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
+import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
+import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
+import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage;
 import org.hyperledger.besu.util.io.RollingFileReader;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.List;
 
 import org.apache.tuweni.bytes.Bytes;
 
@@ -51,26 +59,18 @@ public static void main(final String[] arg) throws IOException {
     final BonsaiWorldState bonsaiState =
         new BonsaiWorldState(
             archive, new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()));
-    //  TODO: fixme and use segmented/composed
-    //    final InMemoryKeyValueStorage accountStorage =
-    //        (InMemoryKeyValueStorage)
-    //
-    // provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
-    //    final InMemoryKeyValueStorage codeStorage =
-    //        (InMemoryKeyValueStorage)
-    //            provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
-    //    final InMemoryKeyValueStorage storageStorage =
-    //        (InMemoryKeyValueStorage)
-    //            provider.getStorageBySegmentIdentifier(
-    //                KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
-    //    final InMemoryKeyValueStorage trieBranchStorage =
-    //        (InMemoryKeyValueStorage)
-    //
-    // provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
-    //    final InMemoryKeyValueStorage trieLogStorage =
-    //        (InMemoryKeyValueStorage)
-    //
-    // provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
+    final SegmentedInMemoryKeyValueStorage worldStateStorage =
+        (SegmentedInMemoryKeyValueStorage)
+            provider.getStorageBySegmentIdentifiers(
+                List.of(
+                    ACCOUNT_INFO_STATE,
+                    CODE_STORAGE,
+                    ACCOUNT_STORAGE_STORAGE,
+                    TRIE_BRANCH_STORAGE));
+
+    final InMemoryKeyValueStorage trieLogStorage =
+        (InMemoryKeyValueStorage)
+            provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
 
     int count = 0;
     while (!reader.isDone()) {
@@ -127,11 +127,7 @@ public static void main(final String[] arg) throws IOException {
       }
     }
     System.out.printf("Back to zero!%n");
-    // TODO: fixme, this is why we held onto allocated in-memory storage in the in mem factory
-    //    accountStorage.dump(System.out);
-    //    codeStorage.dump(System.out);
-    //    storageStorage.dump(System.out);
-    //    trieBranchStorage.dump(System.out);
-    //    trieLogStorage.dump(System.out);
+    worldStateStorage.dump(System.out);
+    trieLogStorage.dump(System.out);
   }
 }
diff --git a/...yperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/...yperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java
@@ -101,7 +101,7 @@ public boolean isSegmentIsolationSupported() {
 
   @Override
   public boolean isSnapshotIsolationSupported() {
-    return publicFactory.isSegmentIsolationSupported();
+    return publicFactory.isSnapshotIsolationSupported();
   }
 
   @Override

diff --git a/...a/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/...a/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java
@@ -189,7 +189,7 @@ public SegmentedKeyValueStorage create(
       case 1, 2 -> {
         if (segmentedStorage == null) {
           final List<SegmentIdentifier> segmentsForVersion =
-              segments.stream()
+              configuredSegments.stream()
                   .filter(segmentId -> segmentId.includeInDatabaseVersion(databaseVersion))
                   .collect(Collectors.toList());
           if (isForestStorageFormat) {

diff --git a/...rc/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBTransaction.java b/...rc/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBTransaction.java
@@ -41,6 +41,7 @@ public class RocksDBTransaction implements SegmentedKeyValueStorageTransaction {
   /**
    * Instantiates a new RocksDb transaction.
    *
+   * @param columnFamilyMapper mapper from segment identifier to column family handle
    * @param innerTx the inner tx
    * @param options the options
    * @param metrics the metrics

diff --git a/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
@@ -116,15 +116,15 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu
    * Instantiates a new Rocks db columnar key value storage.
    *
    * @param configuration the configuration
-   * @param segments the segments
+   * @param defaultSegments the segments
    * @param ignorableSegments the ignorable segments
    * @param metricsSystem the metrics system
    * @param rocksDBMetricsFactory the rocks db metrics factory
    * @throws StorageException the storage exception
    */
   public RocksDBColumnarKeyValueStorage(
       final RocksDBConfiguration configuration,
-      final List<SegmentIdentifier> segments,
+      final List<SegmentIdentifier> defaultSegments,
       final List<SegmentIdentifier> ignorableSegments,
       final MetricsSystem metricsSystem,
       final RocksDBMetricsFactory rocksDBMetricsFactory)
@@ -136,7 +136,7 @@ public RocksDBColumnarKeyValueStorage(
 
     try {
       final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
-      trimmedSegments = new ArrayList<>(segments);
+      trimmedSegments = new ArrayList<>(defaultSegments);
       final List<byte[]> existingColumnFamilies =
           RocksDB.listColumnFamilies(new Options(), configuration.getDatabaseDir().toString());
       // Only ignore if not existed currently
@@ -216,7 +216,7 @@ void initColumnHandles() throws RocksDBException {
                               .filter(
                                   ch -> {
                                     try {
-                                      return ch.getName() == segment.getId();
+                                      return Arrays.equals(ch.getName(), segment.getId());
                                     } catch (RocksDBException e) {
                                       throw new RuntimeException(e);
                                     }
@@ -244,6 +244,12 @@ BlockBasedTableConfig createBlockBasedTableConfig(final RocksDBConfiguration con
         .setBlockSize(ROCKSDB_BLOCK_SIZE);
   }
 
+  /**
+   * Safe method to map segment identifier to column handle.
+   *
+   * @param segment segment identifier
+   * @return column handle
+   */
   protected ColumnFamilyHandle safeColumnHandle(final SegmentIdentifier segment) {
     RocksDbSegmentIdentifier safeRef = columnHandlesBySegmentIdentifier.get(segment);
     if (safeRef == null) {

diff --git a/...yperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java b/...yperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java
@@ -92,6 +92,7 @@ private RocksDBSnapshotTransaction(
   /**
    * Get data against given key.
    *
+   * @param segmentId the segment id
    * @param key the key
    * @return the optional data
    */
@@ -138,6 +139,7 @@ public void remove(final SegmentIdentifier segmentId, final byte[] key) {
   /**
    * Stream.
    *
+   * @param segmentId the segment id
    * @return the stream
    */
   public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentId) {
@@ -152,6 +154,7 @@ public Stream<Pair<byte[], byte[]>> stream(final SegmentIdentifier segmentId) {
   /**
    * Stream keys.
    *
+   * @param segmentId the segment id
    * @return the stream
    */
   public Stream<byte[]> streamKeys(final SegmentIdentifier segmentId) {

diff --git a/.../kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/.../kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java
@@ -16,6 +16,7 @@
 
 import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 
+import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
@@ -60,13 +61,36 @@ private static Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> asSegmentMap
   /** protected access to the rw lock. */
   protected final ReadWriteLock rwLock;
 
+  /**
+   * Instantiates a new In memory key value storage.
+   */
   public InMemoryKeyValueStorage() {
-    super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage());
-    rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock;
+    this(SEGMENT_IDENTIFIER);
   }
 
+  /**
+   * Instantiates a new In memory key value storage with an initial map.
+   * @param initialMap  the initial map
+   */
   public InMemoryKeyValueStorage(final Map<Bytes, Optional<byte[]>> initialMap) {
     super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage(asSegmentMap(initialMap)));
     rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock;
   }
+
+  /**
+   * Instantiates a new In memory key value storage with a single segment identifier.
+   * @param segmentIdentifier the segment identifier
+   */
+  public InMemoryKeyValueStorage(final SegmentIdentifier segmentIdentifier) {
+    super(segmentIdentifier, new SegmentedInMemoryKeyValueStorage());
+    rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock;
+  }
+
+  /**
+   * Dump the contents of the storage to the print stream.
+   * @param ps the print stream.
+   */
+  public void dump(final PrintStream ps) {
+    ((SegmentedInMemoryKeyValueStorage) storage).dump(ps);
+  }
 }
diff --git a/...es/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java b/...es/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java
@@ -25,7 +25,9 @@
 import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,6 +94,8 @@ private void createFactoriesAndRegisterWithStorageService() {
   public static class InMemoryKeyValueStorageFactory implements KeyValueStorageFactory {
 
     private final String name;
+    private final Map<List<SegmentIdentifier>, SegmentedInMemoryKeyValueStorage> storageMap =
+        new HashMap<>();
 
     /**
      * Instantiates a new Memory key value storage factory.
@@ -113,7 +117,10 @@ public KeyValueStorage create(
         final BesuConfiguration configuration,
         final MetricsSystem metricsSystem)
         throws StorageException {
-      return new InMemoryKeyValueStorage();
+      var kvStorage =
+          storageMap.computeIfAbsent(
+              List.of(segment), seg -> new SegmentedInMemoryKeyValueStorage(seg));
+      return new SegmentedKeyValueStorageAdapter(segment, kvStorage);
     }
 
     @Override
@@ -122,7 +129,9 @@ public SegmentedKeyValueStorage create(
         final BesuConfiguration configuration,
         final MetricsSystem metricsSystem)
         throws StorageException {
-      return new SegmentedInMemoryKeyValueStorage();
+      var kvStorage =
+          storageMap.computeIfAbsent(segments, __ -> new SegmentedInMemoryKeyValueStorage());
+      return kvStorage;
     }
 
     @Override
@@ -137,7 +146,7 @@ public boolean isSnapshotIsolationSupported() {
 
     @Override
     public void close() {
-      // Nothing to do
+      storageMap.clear();
     }
   }
 }
diff --git a/...a/org/hyperledger/besu/services/kvstore/KeyValueStorageTransactionValidatorDecorator.java b/...a/org/hyperledger/besu/services/kvstore/KeyValueStorageTransactionValidatorDecorator.java
@@ -32,6 +32,7 @@ public class KeyValueStorageTransactionValidatorDecorator implements KeyValueSto
    * Instantiates a new Key value storage transaction transition validator decorator.
    *
    * @param toDecorate the to decorate
+   * @param isClosed supplier function to determine if the storage is closed
    */
   public KeyValueStorageTransactionValidatorDecorator(
       final KeyValueStorageTransaction toDecorate, final Supplier<Boolean> isClosed) {