Change default test backend to BucketListDB (#4462)

# Description

Resolves #4433

This switches the default testing backend to BucketListDB. While some
test still disable BucketListDB and commit directly via `LedgerTxn` to
test lower level subsystem, such as operation application, the majority
of tests and all end-to-end tests no use BucketListDB. This is rebased
on top of #4455.

# Checklist
- [x] Reviewed the
[contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes)
document
- [x] Rebased on top of master (no merge commits)
- [x] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio
extension)
- [x] Compiles
- [x] Ran all tests
- [ ] If change impacts performance, include supporting evidence per the
[performance
document](https://github.com/stellar/stellar-core/blob/master/performance-eval/performance-eval.md)

src/bucket/Bucket.cpp

@@ -136,14 +136,23 @@ Bucket::apply(Application& app) const
 {
     ZoneScoped;
 
+    auto filter = [&](LedgerEntryType t) {
+        if (app.getConfig().isUsingBucketListDB())
+        {
+            return t == OFFER;
+        }
+
+        return true;
+    };
+
     std::unordered_set<LedgerKey> emptySet;
     BucketApplicator applicator(
         app, app.getConfig().LEDGER_PROTOCOL_VERSION,
         0 /*set to 0 so we always load from the parent to check state*/,
         0 /*set to a level that's not the bottom so we don't treat live entries
              as init*/
         ,
-        shared_from_this(), [](LedgerEntryType) { return true; }, emptySet);
+        shared_from_this(), filter, emptySet);
     BucketApplicator::Counters counters(app.getClock().now());
     while (applicator)
     {

src/bucket/BucketListSnapshot.cpp

@@ -127,6 +127,14 @@ SearchableBucketListSnapshot::getLedgerSeq() const
     return mSnapshot->getLedgerSeq();
 }
 
+LedgerHeader const&
+SearchableBucketListSnapshot::getLedgerHeader()
+{
+    releaseAssert(mSnapshot);
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
+    return mSnapshot->getLedgerHeader();
+}
+
 EvictionResult
 SearchableBucketListSnapshot::scanForEviction(
     uint32_t ledgerSeq, EvictionCounters& counters,

src/bucket/BucketListSnapshot.h

@@ -102,10 +102,6 @@ class SearchableBucketListSnapshot : public NonMovableOrCopyable
                                    std::shared_ptr<EvictionStatistics> stats,
                                    StateArchivalSettings const& sas);
     uint32_t getLedgerSeq() const;
-    LedgerHeader const&
-    getLedgerHeader() const
-    {
-        return mSnapshot->getLedgerHeader();
-    }
+    LedgerHeader const& getLedgerHeader();
 };
 }

src/bucket/test/BucketIndexTests.cpp

@@ -604,7 +604,7 @@ TEST_CASE("ContractData key with same ScVal", "[bucket][bucketindex]")
 
 TEST_CASE("serialize bucket indexes", "[bucket][bucketindex]")
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
 
     // All levels use range config
     cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;

src/bucket/test/BucketListTests.cpp

@@ -148,7 +148,9 @@ TEST_CASE_VERSIONS("bucket list", "[bucket][bucketlist]")
                 lh.ledgerSeq = i;
                 addBatchAndUpdateSnapshot(
                     bl, *app, lh, {},
-                    LedgerTestUtils::generateValidUniqueLedgerEntries(8),
+                    LedgerTestUtils::
+                        generateValidUniqueLedgerEntriesWithExclusions(
+                            {CONFIG_SETTING}, 8),
                     LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions(
                         {CONFIG_SETTING}, 5));
                 if (i % 10 == 0)
@@ -256,7 +258,8 @@ TEST_CASE_VERSIONS("bucket list shadowing pre/post proto 12",
         {
             app->getClock().crank(false);
             auto liveBatch =
-                LedgerTestUtils::generateValidUniqueLedgerEntries(5);
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 5);
 
             BucketEntry BucketEntryAlice, BucketEntryBob;
             alice.balance++;
@@ -351,14 +354,16 @@ TEST_CASE_VERSIONS("bucket tombstones expire at bottom level",
             auto& level = bl.getLevel(i);
             level.setCurr(Bucket::fresh(
                 bm, getAppLedgerVersion(app), {},
-                LedgerTestUtils::generateValidUniqueLedgerEntries(8),
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 8),
                 LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions(
                     {CONFIG_SETTING}, 5),
                 /*countMergeEvents=*/true, clock.getIOContext(),
                 /*doFsync=*/true));
             level.setSnap(Bucket::fresh(
                 bm, getAppLedgerVersion(app), {},
-                LedgerTestUtils::generateValidUniqueLedgerEntries(8),
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 8),
                 LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions(
                     {CONFIG_SETTING}, 5),
                 /*countMergeEvents=*/true, clock.getIOContext(),
@@ -377,7 +382,9 @@ TEST_CASE_VERSIONS("bucket tombstones expire at bottom level",
                 lh.ledgerSeq = j;
                 addBatchAndUpdateSnapshot(
                     bl, *app, lh, {},
-                    LedgerTestUtils::generateValidUniqueLedgerEntries(8),
+                    LedgerTestUtils::
+                        generateValidUniqueLedgerEntriesWithExclusions(
+                            {CONFIG_SETTING}, 8),
                     LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions(
                         {CONFIG_SETTING}, 5));
                 app->getClock().crank(false);
@@ -502,7 +509,9 @@ TEST_CASE_VERSIONS("single entry bubbling up",
             addBatchAndUpdateSnapshot(
                 bl, *app,
                 app->getLedgerManager().getLastClosedLedgerHeader().header, {},
-                LedgerTestUtils::generateValidLedgerEntries(1), emptySet);
+                LedgerTestUtils::generateValidLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 1),
+                emptySet);
 
             CLOG_DEBUG(Bucket, "Adding empty batches to bucket list");
             for (uint32_t i = 2;
@@ -664,7 +673,9 @@ TEST_CASE("BucketList check bucket sizes", "[bucket][bucketlist][count]")
     Application::pointer app = createTestApplication(clock, cfg);
     BucketList& bl = app->getBucketManager().getBucketList();
     std::vector<LedgerKey> emptySet;
-    auto ledgers = LedgerTestUtils::generateValidUniqueLedgerEntries(256);
+    auto ledgers =
+        LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+            {CONFIG_SETTING}, 256);
     for (uint32_t ledgerSeq = 1; ledgerSeq <= 256; ++ledgerSeq)
     {
         if (ledgerSeq >= 2)
@@ -688,7 +699,7 @@ TEST_CASE("BucketList check bucket sizes", "[bucket][bucketlist][count]")
 TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_NO_OFFERS));
     cfg.USE_CONFIG_FOR_GENESIS = true;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);
@@ -760,7 +771,10 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
             }
 
             lm.setNextLedgerEntryBatchForBucketTesting(
-                {}, LedgerTestUtils::generateValidUniqueLedgerEntries(10), {});
+                {},
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 10),
+                {});
             closeLedger(*app);
             if ((ledger + 1) % networkConfig.stateArchivalSettings()
                                    .bucketListWindowSamplePeriod ==
@@ -775,7 +789,7 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
 TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.USE_CONFIG_FOR_GENESIS = true;
 
     auto test = [&](bool backgroundScan) {
@@ -1250,7 +1264,7 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
 TEST_CASE_VERSIONS("Searchable BucketListDB snapshots", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.DEPRECATED_SQL_LEDGER_STATE = false;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);

src/bucket/test/BucketManagerTests.cpp

@@ -213,7 +213,8 @@ TEST_CASE_VERSIONS("bucketmanager ownership", "[bucket][bucketmanager]")
             Application::pointer app = createTestApplication(clock, cfg);
 
             std::vector<LedgerEntry> live(
-                LedgerTestUtils::generateValidUniqueLedgerEntries(10));
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 10));
             std::vector<LedgerKey> dead{};
 
             std::shared_ptr<Bucket> b1;
@@ -303,7 +304,7 @@ TEST_CASE_VERSIONS("bucketmanager ownership", "[bucket][bucketmanager]")
 
 TEST_CASE("bucketmanager missing buckets fail", "[bucket][bucketmanager]")
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     std::string someBucketFileName;
     {
         VirtualClock clock;
@@ -318,7 +319,10 @@ TEST_CASE("bucketmanager missing buckets fail", "[bucket][bucketmanager]")
         {
             ++ledger;
             lm.setNextLedgerEntryBatchForBucketTesting(
-                {}, LedgerTestUtils::generateValidUniqueLedgerEntries(10), {});
+                {},
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 10),
+                {});
             closeLedger(*app);
         } while (!BucketList::levelShouldSpill(ledger, level - 1));
         auto someBucket = bl.getLevel(1).getCurr();
@@ -341,7 +345,7 @@ TEST_CASE_VERSIONS("bucketmanager reattach to finished merge",
                    "[bucket][bucketmanager]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.MANUAL_CLOSE = false;
 
@@ -363,7 +367,9 @@ TEST_CASE_VERSIONS("bucketmanager reattach to finished merge",
             lh.ledgerSeq = ledger;
             addBatchAndUpdateSnapshot(
                 bl, *app, lh, {},
-                LedgerTestUtils::generateValidUniqueLedgerEntries(10), {});
+                LedgerTestUtils::generateValidLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 10),
+                {});
             bm.forgetUnreferencedBuckets();
         } while (!BucketList::levelShouldSpill(ledger, level - 1));
 
@@ -406,7 +412,7 @@ TEST_CASE_VERSIONS("bucketmanager reattach to running merge",
                    "[bucket][bucketmanager]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.MANUAL_CLOSE = false;
 
@@ -450,7 +456,9 @@ TEST_CASE_VERSIONS("bucketmanager reattach to running merge",
             lh.ledgerSeq = ledger;
             addBatchAndUpdateSnapshot(
                 bl, *app, lh, {},
-                LedgerTestUtils::generateValidUniqueLedgerEntries(100), {});
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 100),
+                {});
 
             bm.forgetUnreferencedBuckets();
 
@@ -488,9 +496,10 @@ TEST_CASE("bucketmanager do not leak empty-merge futures",
     // The point of this test is to confirm that
     // BucketManager::noteEmptyMergeOutput is being called properly from merges
     // that produce empty outputs, and that the input buckets to those merges
-    // are thereby not leaking.
+    // are thereby not leaking. Disable BucketListDB so that snapshots do not
+    // hold persist buckets, complicating bucket counting.
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_NO_OFFERS));
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION =
         static_cast<uint32_t>(
@@ -588,7 +597,9 @@ TEST_CASE_VERSIONS(
             lh.ledgerSeq++;
             addBatchAndUpdateSnapshot(
                 bl, *app, lh, {},
-                LedgerTestUtils::generateValidUniqueLedgerEntries(100), {});
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 100),
+                {});
             clock.crank(false);
             bm.forgetUnreferencedBuckets();
         }
@@ -1064,7 +1075,7 @@ class StopAndRestartBucketMergesTest
     collectControlSurveys()
     {
         VirtualClock clock;
-        Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+        Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
         cfg.ARTIFICIALLY_REDUCE_MERGE_COUNTS_FOR_TESTING = true;
         cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = mProtocol;
@@ -1175,10 +1186,11 @@ class StopAndRestartBucketMergesTest
     runStopAndRestartTest(uint32_t firstProtocol, uint32_t secondProtocol)
     {
         std::unique_ptr<VirtualClock> clock = std::make_unique<VirtualClock>();
-        Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+        Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
         cfg.ARTIFICIALLY_REDUCE_MERGE_COUNTS_FOR_TESTING = true;
         cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = firstProtocol;
+        cfg.INVARIANT_CHECKS = {};
         assert(!mDesignatedLedgers.empty());
         uint32_t finalLedger = (*mDesignatedLedgers.rbegin()) + 1;
         uint32_t currProtocol = firstProtocol;
@@ -1367,17 +1379,18 @@ TEST_CASE_VERSIONS("bucket persistence over app restart",
     std::vector<stellar::LedgerKey> emptySet;
     std::vector<stellar::LedgerEntry> emptySetEntry;
 
-    Config cfg0(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg0(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     cfg0.MANUAL_CLOSE = false;
 
     for_versions_with_differing_bucket_logic(cfg0, [&](Config const& cfg0) {
-        Config cfg1(getTestConfig(1, Config::TESTDB_ON_DISK_SQLITE));
+        Config cfg1(getTestConfig(1, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg1.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION =
             cfg0.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION;
         cfg1.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
 
         auto batch_entries =
-            LedgerTestUtils::generateValidUniqueLedgerEntries(111);
+            LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                {CONFIG_SETTING, OFFER}, 111);
         auto alice = batch_entries.back();
         batch_entries.pop_back();
         std::vector<std::vector<LedgerEntry>> batches;

src/bucket/test/BucketMergeMapTests.cpp

@@ -20,7 +20,8 @@ TEST_CASE("bucket merge map", "[bucket][bucketmergemap]")
 
     auto getValidBucket = [&](int numEntries = 10) {
         std::vector<LedgerEntry> live =
-            LedgerTestUtils::generateValidUniqueLedgerEntries(numEntries);
+            LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                {CONFIG_SETTING}, numEntries);
         std::shared_ptr<Bucket> b1 = Bucket::fresh(
             app->getBucketManager(), BucketTestUtils::getAppLedgerVersion(app),
             {}, live, {},

src/bucket/test/BucketTests.cpp

@@ -223,7 +223,8 @@ TEST_CASE_VERSIONS("merging bucket entries", "[bucket]")
         SECTION("random live entries overwrite live entries in any order")
         {
             std::vector<LedgerEntry> live =
-                LedgerTestUtils::generateValidUniqueLedgerEntries(100);
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 100);
             std::vector<LedgerKey> dead;
             std::shared_ptr<Bucket> b1 = Bucket::fresh(
                 app->getBucketManager(), getAppLedgerVersion(app), {}, live,
@@ -910,10 +911,10 @@ TEST_CASE_VERSIONS("merging bucket entries with initentry with shadows",
     });
 }
 
-TEST_CASE_VERSIONS("bucket apply", "[bucket]")
+TEST_CASE_VERSIONS("legacy bucket apply", "[bucket]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig());
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_OFFERS));
     for_versions_with_differing_bucket_logic(cfg, [&](Config const& cfg) {
         Application::pointer app = createTestApplication(clock, cfg);
 
@@ -983,14 +984,5 @@ TEST_CASE("bucket apply bench", "[bucketbench][!hide]")
         birth->apply(*app);
     };
 
-    SECTION("sqlite")
-    {
-        runtest(Config::TESTDB_ON_DISK_SQLITE);
-    }
-#ifdef USE_POSTGRES
-    SECTION("postgresql")
-    {
-        runtest(Config::TESTDB_POSTGRESQL);
-    }
-#endif
+    runtest(Config::TESTDB_BUCKET_DB_PERSISTENT);
 }

src/database/test/DatabaseTests.cpp

@@ -72,7 +72,7 @@ transactionTest(Application::pointer app)
 
 TEST_CASE("database smoketest", "[db]")
 {
-    Config const& cfg = getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE);
+    Config const& cfg = getTestConfig(0, Config::TESTDB_IN_MEMORY_OFFERS);
 
     VirtualClock clock;
     Application::pointer app = createTestApplication(clock, cfg, true, false);
@@ -349,7 +349,7 @@ TEST_CASE("postgres performance", "[db][pgperf][!hide]")
 
 TEST_CASE("schema test", "[db]")
 {
-    Config const& cfg = getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE);
+    Config const& cfg = getTestConfig(0, Config::TESTDB_IN_MEMORY_OFFERS);
 
     VirtualClock clock;
     Application::pointer app = createTestApplication(clock, cfg);