Add BONSAI_ARCHIVE storage format

Signed-off-by: Matthew Whitehead <[email protected]>

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1992,7 +1992,7 @@ private PrivacyParameters privacyParameters() {
         throw new ParameterException(
             commandLine, String.format("%s %s", "Checkpoint sync", errorSuffix));
       }
-      if (getDataStorageConfiguration().getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+      if (getDataStorageConfiguration().getDataStorageFormat().isBonsaiFormat()) {
         throw new ParameterException(commandLine, String.format("%s %s", "Bonsai", errorSuffix));
       }
 

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -48,7 +48,7 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
   @Option(
       names = {DATA_STORAGE_FORMAT},
       description =
-          "Format to store trie data in.  Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).",
+          "Format to store trie data in.  Either FOREST, BONSAI or BONSAI_ARCHIVE (default: ${DEFAULT-VALUE}).",
       arity = "1")
   private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI;
 

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java

@@ -123,6 +123,7 @@ public void run() {
             switch (dataStorageFormat) {
               case FOREST -> 1;
               case BONSAI -> 2;
+              case BONSAI_ARCHIVE -> 3;
             };
 
         @JsonSerialize

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -28,8 +28,6 @@
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
-import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
-import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -323,8 +321,8 @@ private static TrieLogContext getTrieLogContext() {
     BesuController besuController = createBesuController();
     final DataStorageConfiguration config = besuController.getDataStorageConfiguration();
     checkArgument(
-        DataStorageFormat.BONSAI.equals(config.getDataStorageFormat()),
-        "Subcommand only works with data-storage-format=BONSAI");
+        config.getDataStorageFormat().isBonsaiFormat(),
+        "Subcommand only works with data-storage-format=BONSAI or BONSAI_ARCHIVE");
 
     final StorageProvider storageProvider = besuController.getStorageProvider();
     final BonsaiWorldStateKeyValueStorage rootWorldStateStorage =

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -1090,6 +1090,15 @@ WorldStateArchive createWorldStateArchive(
       final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) {
     return switch (dataStorageConfiguration.getDataStorageFormat()) {
       case BONSAI -> {
+        yield new BonsaiWorldStateProvider(
+            worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class),
+            blockchain,
+            Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
+            bonsaiCachedMerkleTrieLoader,
+            besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
+            evmConfiguration);
+      }
+      case BONSAI_ARCHIVE -> {
         final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
             worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.ethereum.storage.keyvalue;
 
 import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI;
+import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI_ARCHIVE;
 import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST;
 
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
@@ -30,11 +31,17 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   PRIVATE_TRANSACTIONS(new byte[] {3}),
   PRIVATE_STATE(new byte[] {4}),
   PRUNING_STATE(new byte[] {5}, EnumSet.of(FOREST)),
-  ACCOUNT_INFO_STATE(new byte[] {6}, EnumSet.of(BONSAI), false, true, false),
-  CODE_STORAGE(new byte[] {7}, EnumSet.of(BONSAI)),
-  ACCOUNT_STORAGE_STORAGE(new byte[] {8}, EnumSet.of(BONSAI), false, true, false),
-  TRIE_BRANCH_STORAGE(new byte[] {9}, EnumSet.of(BONSAI), false, true, false),
-  TRIE_LOG_STORAGE(new byte[] {10}, EnumSet.of(BONSAI), true, false, true),
+  ACCOUNT_INFO_STATE(new byte[] {6}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), false, true, false),
+  CODE_STORAGE(new byte[] {7}, EnumSet.of(BONSAI, BONSAI_ARCHIVE)),
+  ACCOUNT_STORAGE_STORAGE(new byte[] {8}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), false, true, false),
+  TRIE_BRANCH_STORAGE(new byte[] {9}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), false, true, false),
+  TRIE_LOG_STORAGE(new byte[] {10}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), true, false, true),
+  ACCOUNT_FREEZER_STATE(
+      "ACCOUNT_FREEZER_STATE".getBytes(StandardCharsets.UTF_8),
+      EnumSet.of(BONSAI_ARCHIVE),
+      true,
+      false,
+      true),
   VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE
 
   // previously supported GoQuorum private states

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java

@@ -26,7 +26,6 @@
 import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
-import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
@@ -82,7 +81,7 @@ public BlockchainStorage createBlockchainStorage(
   @Override
   public WorldStateKeyValueStorage createWorldStateStorage(
       final DataStorageConfiguration dataStorageConfiguration) {
-    if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+    if (dataStorageConfiguration.getDataStorageFormat().isBonsaiFormat()) {
       return new BonsaiWorldStateKeyValueStorage(this, metricsSystem, dataStorageConfiguration);
     } else {
       return new ForestWorldStateKeyValueStorage(

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java

@@ -28,7 +28,6 @@
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
-import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage;
 
@@ -44,8 +43,7 @@ public class GenesisWorldStateProvider {
    */
   public static MutableWorldState createGenesisWorldState(
       final DataStorageConfiguration dataStorageConfiguration) {
-    if (Objects.requireNonNull(dataStorageConfiguration).getDataStorageFormat()
-        == DataStorageFormat.BONSAI) {
+    if (Objects.requireNonNull(dataStorageConfiguration).getDataStorageFormat().isBonsaiFormat()) {
       return createGenesisBonsaiWorldState();
     } else {
       return createGenesisForestWorldState();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java

@@ -19,6 +19,7 @@
 import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -74,7 +75,7 @@ public <STRATEGY extends WorldStateKeyValueStorage> STRATEGY getStrategy(
   }
 
   public boolean isMatchingFlatMode(final FlatDbMode flatDbMode) {
-    if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+    if (getDataStorageFormat().isBonsaiFormat()) {
       final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy =
           (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage();
       return bonsaiWorldStateStorageStrategy.getFlatDbMode().equals(flatDbMode);
@@ -84,8 +85,8 @@ public boolean isMatchingFlatMode(final FlatDbMode flatDbMode) {
 
   public void applyOnMatchingFlatMode(
       final FlatDbMode flatDbMode, final Consumer<BonsaiWorldStateKeyValueStorage> onStrategy) {
-    applyOnMatchingStrategy(
-        DataStorageFormat.BONSAI,
+    applyOnMatchingStrategies(
+        List.of(DataStorageFormat.BONSAI, DataStorageFormat.BONSAI_ARCHIVE),
         worldStateKeyValueStorage -> {
           final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy =
               (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage();
@@ -96,8 +97,8 @@ public void applyOnMatchingFlatMode(
   }
 
   public void applyWhenFlatModeEnabled(final Consumer<BonsaiWorldStateKeyValueStorage> onStrategy) {
-    applyOnMatchingStrategy(
-        DataStorageFormat.BONSAI,
+    applyOnMatchingStrategies(
+        List.of(DataStorageFormat.BONSAI, DataStorageFormat.BONSAI_ARCHIVE),
         worldStateKeyValueStorage -> {
           final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy =
               (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage();
@@ -115,10 +116,18 @@ public void applyOnMatchingStrategy(
     }
   }
 
+  public void applyOnMatchingStrategies(
+      final List<DataStorageFormat> dataStorageFormats,
+      final Consumer<WorldStateKeyValueStorage> onStrategy) {
+    if (dataStorageFormats.contains(getDataStorageFormat())) {
+      onStrategy.accept(worldStateKeyValueStorage());
+    }
+  }
+
   public <RESPONSE> RESPONSE applyForStrategy(
       final Function<BonsaiWorldStateKeyValueStorage, RESPONSE> onBonsai,
       final Function<ForestWorldStateKeyValueStorage, RESPONSE> onForest) {
-    if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+    if (getDataStorageFormat().isBonsaiFormat()) {
       return onBonsai.apply(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage()));
     } else {
       return onForest.apply(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage()));
@@ -128,7 +137,7 @@ public <RESPONSE> RESPONSE applyForStrategy(
   public void consumeForStrategy(
       final Consumer<BonsaiWorldStateKeyValueStorage> onBonsai,
       final Consumer<ForestWorldStateKeyValueStorage> onForest) {
-    if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+    if (getDataStorageFormat().isBonsaiFormat()) {
       onBonsai.accept(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage()));
     } else {
       onForest.accept(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage()));

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java

@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
@@ -85,8 +86,8 @@ public CompletableFuture<FastSyncState> start() {
   }
 
   protected CompletableFuture<FastSyncState> start(final FastSyncState fastSyncState) {
-    worldStateStorageCoordinator.applyOnMatchingStrategy(
-        DataStorageFormat.BONSAI,
+    worldStateStorageCoordinator.applyOnMatchingStrategies(
+        List.of(DataStorageFormat.BONSAI, DataStorageFormat.BONSAI_ARCHIVE),
         worldStateKeyValueStorage -> {
           BonsaiWorldStateKeyValueStorage onBonsai =
               (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage;

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java

@@ -254,8 +254,8 @@ public synchronized void startTrieHeal() {
   /** Method to reload the healing process of the trie */
   public synchronized void reloadTrieHeal() {
     // Clear the flat database and trie log from the world state storage if needed
-    worldStateStorageCoordinator.applyOnMatchingStrategy(
-        DataStorageFormat.BONSAI,
+    worldStateStorageCoordinator.applyOnMatchingStrategies(
+        List.of(DataStorageFormat.BONSAI, DataStorageFormat.BONSAI_ARCHIVE),
         worldStateKeyValueStorage -> {
           final BonsaiWorldStateKeyValueStorage strategy =
               worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java

@@ -169,8 +169,8 @@ public CompletableFuture<Void> run(
                 });
       } else if (!snapContext.getAccountsHealingList().isEmpty()) { // restart only the heal step
         snapSyncState.setHealTrieStatus(true);
-        worldStateStorageCoordinator.applyOnMatchingStrategy(
-            DataStorageFormat.BONSAI,
+        worldStateStorageCoordinator.applyOnMatchingStrategies(
+            List.of(DataStorageFormat.BONSAI, DataStorageFormat.BONSAI_ARCHIVE),
             strategy -> {
               BonsaiWorldStateKeyValueStorage onBonsai = (BonsaiWorldStateKeyValueStorage) strategy;
               onBonsai.clearFlatDatabase();

plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageFormat.java

@@ -19,5 +19,11 @@ public enum DataStorageFormat {
   /** Original format. Store all tries */
   FOREST,
   /** New format. Store one trie, and trie logs to roll forward and backward */
-  BONSAI;
+  BONSAI,
+  /** The new option for storing archive data e.g. state at any block */
+  BONSAI_ARCHIVE;
+
+  public boolean isBonsaiFormat() {
+    return this == BONSAI || this == BONSAI_ARCHIVE;
+  }
 }

plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.plugin.services.storage.rocksdb;
 
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION;
 import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION;
 import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES;
 import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION;
@@ -57,7 +58,10 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory {
 
   private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValueStorageFactory.class);
   private static final EnumSet<BaseVersionedStorageFormat> SUPPORTED_VERSIONED_FORMATS =
-      EnumSet.of(FOREST_WITH_RECEIPT_COMPACTION, BONSAI_WITH_RECEIPT_COMPACTION);
+      EnumSet.of(
+          FOREST_WITH_RECEIPT_COMPACTION,
+          BONSAI_WITH_RECEIPT_COMPACTION,
+          BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION);
   private static final String NAME = "rocksdb";
   private final RocksDBMetricsFactory rocksDBMetricsFactory;
   private DatabaseMetadata databaseMetadata;
@@ -160,7 +164,7 @@ public SegmentedKeyValueStorage create(
                   metricsSystem,
                   rocksDBMetricsFactory);
         }
-        case BONSAI -> {
+        case BONSAI, BONSAI_ARCHIVE -> {
           LOG.debug("BONSAI mode detected, Using OptimisticTransactionDB.");
           segmentedStorage =
               new OptimisticRocksDBColumnarKeyValueStorage(

plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java

@@ -44,7 +44,17 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat {
    * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk
    * space
    */
-  BONSAI_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI, 3);
+  BONSAI_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI, 3),
+  /**
+   * Current Bonsai version, with blockchain variables in a dedicated column family, in order to
+   * make BlobDB more effective
+   */
+  BONSAI_ARCHIVE_WITH_VARIABLES(DataStorageFormat.BONSAI_ARCHIVE, 1),
+  /**
+   * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk
+   * space
+   */
+  BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI_ARCHIVE, 2);
 
   private final DataStorageFormat format;
   private final int version;
@@ -65,6 +75,7 @@ public static BaseVersionedStorageFormat defaultForNewDB(
     return switch (configuration.getDatabaseFormat()) {
       case FOREST -> FOREST_WITH_RECEIPT_COMPACTION;
       case BONSAI -> BONSAI_WITH_RECEIPT_COMPACTION;
+      case BONSAI_ARCHIVE -> BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION;
     };
   }
 

plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java

@@ -239,7 +239,7 @@ public DatabaseMetadata upgradeToPrivacy() {
                         "Unsupported database with format FOREST and version "
                             + versionedStorageFormat.getVersion());
               };
-          case BONSAI ->
+          case BONSAI, BONSAI_ARCHIVE ->
               switch (versionedStorageFormat.getVersion()) {
                 case 1 -> PrivacyVersionedStorageFormat.BONSAI_ORIGINAL;
                 case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES;

plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java

@@ -44,7 +44,13 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat {
    * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk
    * space
    */
-  BONSAI_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION, 2);
+  BONSAI_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION, 2),
+  /**
+   * Bonsai archive version, with receipts using compaction, in order to make Receipts use less disk
+   * space
+   */
+  BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION(
+      BaseVersionedStorageFormat.BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION, 3);
 
   private final VersionedStorageFormat baseVersionedStorageFormat;
   private final OptionalInt privacyVersion;
@@ -66,6 +72,7 @@ public static VersionedStorageFormat defaultForNewDB(
     return switch (configuration.getDatabaseFormat()) {
       case FOREST -> FOREST_WITH_RECEIPT_COMPACTION;
       case BONSAI -> BONSAI_WITH_RECEIPT_COMPACTION;
+      case BONSAI_ARCHIVE -> BONSAI_ARCHIVE_WITH_RECEIPT_COMPACTION;
     };
   }
 

plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/Utils.java

@@ -86,6 +86,7 @@ private static int dataStorageFormatToV1(final DataStorageFormat dataStorageForm
     return switch (dataStorageFormat) {
       case FOREST -> 1;
       case BONSAI -> 2;
+      case BONSAI_ARCHIVE -> 3;
     };
   }
 }