diff --git a/docs/stellar-core_example.cfg b/docs/stellar-core_example.cfg index 4620177a82..103c115cf4 100644 --- a/docs/stellar-core_example.cfg +++ b/docs/stellar-core_example.cfg @@ -634,6 +634,10 @@ EMIT_SOROBAN_TRANSACTION_META_EXT_V1=false # https://github.com/stellar/stellar-xdr/commit/cdc339f5e74a75e8e558fd1a853397da71f1659a EMIT_LEDGER_CLOSE_META_EXT_V1=false +# When set to true, Core will revert to using the old, application-agnostic +# nomination weight function for SCP leader election. +FORCE_OLD_STYLE_LEADER_ELECTION=false + # EXCLUDE_TRANSACTIONS_CONTAINING_OPERATION_TYPE (list of strings) default is empty # Setting this will cause the node to reject transactions that it receives if # they contain any operation in this list. It will not, however, stop the node diff --git a/src/herder/HerderSCPDriver.cpp b/src/herder/HerderSCPDriver.cpp index 756402869f..a46a6a8b21 100644 --- a/src/herder/HerderSCPDriver.cpp +++ b/src/herder/HerderSCPDriver.cpp @@ -25,6 +25,7 @@ #include "xdr/Stellar-ledger.h" #include #include +#include #include #include #include @@ -1284,4 +1285,71 @@ HerderSCPDriver::TxSetValidityKeyHash::operator()( hashMix(res, std::get<3>(key)); return res; } + +uint64 +HerderSCPDriver::getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset, + bool const isLocalNode) const +{ + Config const& cfg = mApp.getConfig(); + bool const unsupportedProtocol = protocolVersionIsBefore( + mApp.getLedgerManager() + .getLastClosedLedgerHeader() + .header.ledgerVersion, + APPLICATION_SPECIFIC_NOMINATION_LEADER_ELECTION_PROTOCOL_VERSION); + if (unsupportedProtocol || !cfg.VALIDATOR_WEIGHT_CONFIG.has_value() || + cfg.FORCE_OLD_STYLE_LEADER_ELECTION) + { + // Fall back on old weight algorithm if any of the following are true: + // 1. The network has not yet upgraded to + // APPLICATION_SPECIFIC_NOMINATION_LEADER_ELECTION_PROTOCOL_VERSION, + // 2. The node is using manual quorum set configuration, or + // 3. The node has the FORCE_OLD_STYLE_LEADER_ELECTION flag + // set + return SCPDriver::getNodeWeight(nodeID, qset, isLocalNode); + } + + ValidatorWeightConfig const& vwc = + mApp.getConfig().VALIDATOR_WEIGHT_CONFIG.value(); + + auto entryIt = vwc.mValidatorEntries.find(nodeID); + if (entryIt == vwc.mValidatorEntries.end()) + { + // This shouldn't be possible as the validator entries should contain + // all validators in the config. For this to happen, `getNodeWeight` + // would have to be called with a non-validator `nodeID`. Throw if + // building tests, and otherwise fall back on the old algorithm. + throw std::runtime_error( + fmt::format(FMT_STRING("Validator entry not found for node {}"), + toShortString(nodeID))); + } + + ValidatorEntry const& entry = entryIt->second; + auto homeDomainSizeIt = vwc.mHomeDomainSizes.find(entry.mHomeDomain); + if (homeDomainSizeIt == vwc.mHomeDomainSizes.end()) + { + // This shouldn't be possible as the home domain sizes should contain + // all home domains in the config. For this to happen, `getNodeWeight` + // would have to be called with a non-validator, or the config parser + // would have to allow a validator without a home domain. Throw if + // building tests, and otherwise fall back on the old algorithm. + throw std::runtime_error( + fmt::format(FMT_STRING("Home domain size not found for domain {}"), + entry.mHomeDomain)); + } + + auto qualityWeightIt = vwc.mQualityWeights.find(entry.mQuality); + if (qualityWeightIt == vwc.mQualityWeights.end()) + { + // This shouldn't be possible as the quality weights should contain all + // quality levels in the config. + throw std::runtime_error( + fmt::format(FMT_STRING("Quality weight not found for quality {}"), + static_cast(entry.mQuality))); + } + + // Node's weight is its quality's weight divided by the number of nodes in + // its home domain + releaseAssert(homeDomainSizeIt->second > 0); + return qualityWeightIt->second / homeDomainSizeIt->second; +} } diff --git a/src/herder/HerderSCPDriver.h b/src/herder/HerderSCPDriver.h index 2d8a998681..f5f74e0365 100644 --- a/src/herder/HerderSCPDriver.h +++ b/src/herder/HerderSCPDriver.h @@ -9,6 +9,7 @@ #include "herder/TxSetUtils.h" #include "medida/timer.h" #include "scp/SCPDriver.h" +#include "util/ProtocolVersion.h" #include "util/RandomEvictionCache.h" #include "xdr/Stellar-ledger.h" #include @@ -32,6 +33,11 @@ class VirtualTimer; struct StellarValue; struct SCPEnvelope; +// First protocol version supporting the application-specific weight function +// for SCP leader election. +ProtocolVersion constexpr APPLICATION_SPECIFIC_NOMINATION_LEADER_ELECTION_PROTOCOL_VERSION = + ProtocolVersion::V_22; + class HerderSCPDriver : public SCPDriver { public: @@ -129,6 +135,14 @@ class HerderSCPDriver : public SCPDriver Json::Value getQsetLagInfo(bool summary, bool fullKeys); + // Application-specific weight function. This function uses the quality + // levels from automatic quorum set generation to determine the weight of a + // validator. It is designed to ensure that: + // 1. Orgs of equal quality have equal chances of winning leader election. + // 2. Higher quality orgs win more frequently than lower quality orgs. + uint64 getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset, + bool isLocalNode) const override; + private: Application& mApp; HerderImpl& mHerder; diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index b5e6bc8f1d..059b0795bc 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -8,6 +8,7 @@ #include "main/Application.h" #include "main/Config.h" #include "scp/SCP.h" +#include "scp/Slot.h" #include "simulation/Simulation.h" #include "simulation/Topologies.h" #include "test/TestAccount.h" @@ -45,6 +46,7 @@ #include "xdrpp/autocheck.h" #include "xdrpp/marshal.h" #include +#include #include #include #include @@ -5562,3 +5564,500 @@ TEST_CASE("SCP message capture from previous ledger", "[herder]") // closing ledger 3. REQUIRE(checkSCPHistoryEntries(C, 2, expectedTypes)); } + +using Topology = std::pair, std::vector>; + +// Generate a Topology with a single org containing 3 validators of HIGH quality +static Topology +simpleThreeNode() +{ + // Generate validators + std::vector sks; + std::vector validators; + int constexpr numValidators = 3; + for (int i = 0; i < numValidators; ++i) + { + SecretKey const& key = sks.emplace_back(SecretKey::random()); + ValidatorEntry& entry = validators.emplace_back(); + entry.mName = fmt::format("validator-{}", i); + entry.mHomeDomain = "A"; + entry.mQuality = ValidatorQuality::VALIDATOR_HIGH_QUALITY; + entry.mKey = key.getPublicKey(); + entry.mHasHistory = false; + } + return {sks, validators}; +} + +// Generate a topology with 3 orgs of HIGH quality. Two orgs have 3 validators +// and one org has 5 validators. +static Topology +unbalancedOrgs() +{ + // Generate validators + std::vector sks; + std::vector validators; + int constexpr numValidators = 11; + for (int i = 0; i < numValidators; ++i) + { + // Orgs A and B have 3 validators each. Org C has 5 validators. + std::string org = "C"; + if (i < 3) + { + org = "A"; + } + else if (i < 6) + { + org = "B"; + } + + SecretKey const& key = sks.emplace_back(SecretKey::random()); + ValidatorEntry& entry = validators.emplace_back(); + entry.mName = fmt::format("validator-{}", i); + entry.mHomeDomain = org; + entry.mQuality = ValidatorQuality::VALIDATOR_HIGH_QUALITY; + entry.mKey = key.getPublicKey(); + entry.mHasHistory = false; + } + return {sks, validators}; +} + +// Generate a tier1-like topology. This topology has 7 HIGH quality orgs, 6 of +// which have 3 validators and 1 has 5 validators. +static Topology +teir1Like() +{ + std::vector sks; + std::vector validators; + int constexpr numOrgs = 7; + + for (int i = 0; i < numOrgs; ++i) + { + std::string const org = fmt::format("org-{}", i); + int const numValidators = i == 0 ? 5 : 3; + for (int j = 0; j < numValidators; ++j) + { + SecretKey const& key = sks.emplace_back(SecretKey::random()); + ValidatorEntry& entry = validators.emplace_back(); + entry.mName = fmt::format("validator-{}-{}", i, j); + entry.mHomeDomain = org; + entry.mQuality = ValidatorQuality::VALIDATOR_HIGH_QUALITY; + entry.mKey = key.getPublicKey(); + entry.mHasHistory = false; + } + } + + return {sks, validators}; +} + +// Returns a random quality up to `maxQuality` +static ValidatorQuality +randomQuality(ValidatorQuality maxQuality) +{ + return static_cast(rand_uniform( + static_cast(ValidatorQuality::VALIDATOR_LOW_QUALITY), + static_cast(maxQuality))); +} + +// Returns the minimum size an org of quality `q` can have +static int constexpr minOrgSize(ValidatorQuality q) +{ + switch (q) + { + case ValidatorQuality::VALIDATOR_LOW_QUALITY: + case ValidatorQuality::VALIDATOR_MED_QUALITY: + return 1; + case ValidatorQuality::VALIDATOR_HIGH_QUALITY: + case ValidatorQuality::VALIDATOR_CRITICAL_QUALITY: + return 3; + } +} + +// Generate a random topology with up to `maxValidators` validators. Ensures at +// least one org is HIGH quality. +static Topology +randomTopology(int maxValidators) +{ + int const numValidators = rand_uniform(3, maxValidators); + int constexpr minCritOrgSize = + minOrgSize(ValidatorQuality::VALIDATOR_CRITICAL_QUALITY); + + // Generate validators + int curOrg = 0; + int curOrgSize = 0; + ValidatorQuality curQuality = ValidatorQuality::VALIDATOR_HIGH_QUALITY; + std::vector sks(numValidators); + std::vector validators(numValidators); + for (int i = 0; i < numValidators; ++i) + { + if (curOrgSize >= minOrgSize(curQuality) && rand_flip()) + { + // Start new org + ++curOrg; + curOrgSize = 0; + curQuality = + randomQuality(numValidators - i >= minCritOrgSize + ? ValidatorQuality::VALIDATOR_CRITICAL_QUALITY + : ValidatorQuality::VALIDATOR_MED_QUALITY); + } + + std::string const org = fmt::format("org-{}", curOrg); + SecretKey const& key = sks.at(i) = SecretKey::random(); + + ValidatorEntry& entry = validators.at(i); + entry.mName = fmt::format("validator-{}", i); + entry.mHomeDomain = org; + entry.mQuality = curQuality; + entry.mKey = key.getPublicKey(); + entry.mHasHistory = false; + + ++curOrgSize; + } + + return {sks, validators}; +} + +// Expected weight of an org with quality `orgQuality` in a topology with a max +// quality of `maxQuality` and or quality counts of `orgQualityCounts`. This +// function normalizes the weight so that the highest quality has a weight of +// `1`. +static double +expectedOrgNormalizedWeight( + std::unordered_map const& orgQualityCounts, + ValidatorQuality maxQuality, ValidatorQuality orgQuality) +{ + if (orgQuality == ValidatorQuality::VALIDATOR_LOW_QUALITY) + { + return 0.0; + } + + double normalizedWeight = 1.0; + + // For each quality level higher than `orgQuality`, divide the weight by 10 + // times the number of orgs at that quality level + for (int q = static_cast(maxQuality); q > static_cast(orgQuality); + --q) + { + normalizedWeight /= + 10 * orgQualityCounts.at(static_cast(q)); + } + return normalizedWeight; +} + +// Expected weight of a validator in an org of size `orgSize` with quality +// `orgQuality`. `maxQuality` is the maximum quality present in the +// configuration. This function normalizes the weight so that the highest +// organization-level quality has a weight of `1`. +static double +expectedNormalizedWeight( + std::unordered_map const& orgQualityCounts, + ValidatorQuality maxQuality, ValidatorQuality orgQuality, int orgSize) +{ + return expectedOrgNormalizedWeight(orgQualityCounts, maxQuality, + orgQuality) / + orgSize; +} + +// Collect information about the qualities and sizes of organizations in +// `validators` and store them in `maxQuality`, `orgQualities`, `orgSizes`, and +// `orgQualityCounts`. +static void +collectOrgInfo(ValidatorQuality& maxQuality, + std::unordered_map& orgQualities, + std::unordered_map& orgSizes, + std::unordered_map& orgQualityCounts, + std::vector const& validators) +{ + maxQuality = ValidatorQuality::VALIDATOR_LOW_QUALITY; + ValidatorQuality minQuality = ValidatorQuality::VALIDATOR_CRITICAL_QUALITY; + std::unordered_map> + orgsByQuality; + for (ValidatorEntry const& validator : validators) + { + maxQuality = std::max(maxQuality, validator.mQuality); + minQuality = std::min(minQuality, validator.mQuality); + orgQualities[validator.mHomeDomain] = validator.mQuality; + ++orgSizes[validator.mHomeDomain]; + orgsByQuality[validator.mQuality].insert(validator.mHomeDomain); + } + + // Count orgs at each quality level + for (int q = static_cast(minQuality); + q <= static_cast(maxQuality); ++q) + { + orgQualityCounts[static_cast(q)] = + orgsByQuality[static_cast(q)].size(); + if (q != static_cast(minQuality)) + { + // Add virtual org covering next lower quality level + ++orgQualityCounts[static_cast(q)]; + } + } +} + +// Given a list of validators, test that the weights of the validators herder +// reports are correct +static void +testWeights(std::vector const& validators) +{ + Config cfg = getTestConfig(0); + + cfg.generateQuorumSetForTesting(validators); + + VirtualClock clock; + Application::pointer app = createTestApplication(clock, cfg); + for_versions_from( + static_cast( + APPLICATION_SPECIFIC_NOMINATION_LEADER_ELECTION_PROTOCOL_VERSION), + *app, [&]() { + // Collect info about orgs + ValidatorQuality maxQuality; + std::unordered_map orgQualities; + std::unordered_map orgSizes; + std::unordered_map orgQualityCounts; + collectOrgInfo(maxQuality, orgQualities, orgSizes, orgQualityCounts, + validators); + + // Check per-validator weights + HerderImpl& herder = dynamic_cast(app->getHerder()); + std::unordered_map normalizedOrgWeights; + for (ValidatorEntry const& validator : validators) + { + uint64_t weight = herder.getHerderSCPDriver().getNodeWeight( + validator.mKey, cfg.QUORUM_SET, false); + double normalizedWeight = + static_cast(weight) / UINT64_MAX; + normalizedOrgWeights[validator.mHomeDomain] += normalizedWeight; + + std::string const& org = validator.mHomeDomain; + REQUIRE_THAT(normalizedWeight, + Catch::Matchers::WithinAbs( + expectedNormalizedWeight( + orgQualityCounts, maxQuality, + orgQualities.at(org), orgSizes.at(org)), + 0.0001)); + } + + // Check per-org weights + for (auto const& [org, weight] : normalizedOrgWeights) + { + REQUIRE_THAT(weight, Catch::Matchers::WithinAbs( + expectedOrgNormalizedWeight( + orgQualityCounts, maxQuality, + orgQualities.at(org)), + 0.0001)); + } + }); +} + +// Test that HerderSCPDriver::getNodeWeight produces weights that result in a +// fair distribution of nomination wins. +TEST_CASE_VERSIONS("getNodeWeight", "[herder]") +{ + SECTION("3 tier 1 validators, 1 org") + { + testWeights(simpleThreeNode().second); + } + + SECTION("11 tier 1 validators, 3 unbalanced orgs") + { + testWeights(unbalancedOrgs().second); + } + + SECTION("Tier1-like topology") + { + testWeights(teir1Like().second); + } + + SECTION("Random topology") + { + // Test weights for 1000 random topologies of up to 200 validators + for (int i = 0; i < 1000; ++i) + { + testWeights(randomTopology(200).second); + } + } +} + +static Value +getRandomValue() +{ + auto h = sha256(fmt::format("value {}", gRandomEngine())); + return xdr::xdr_to_opaque(h); +} + +// A test version of NominationProtocol that exposes `updateRoundLeaders` +class TestNominationProtocol : public NominationProtocol +{ + public: + TestNominationProtocol(Slot& slot) : NominationProtocol(slot) + { + } + + std::set const& + updateRoundLeadersForTesting() + { + mPreviousValue = getRandomValue(); + updateRoundLeaders(); + return getLeaders(); + } + + // Detect fast timeouts by examining the final round number + bool + fastTimedOut() const + { + return mRoundNumber > 0; + } +}; + +// Test nomination over `numLedgers` slots. After running, check that the win +// percentages of each node and org are within 5% of the expected win +// percentages. +static void +testWinProbabilities(std::vector const& sks, + std::vector const& validators, + int const numLedgers) +{ + REQUIRE(sks.size() == validators.size()); + + // Collect info about orgs + ValidatorQuality maxQuality; + std::unordered_map orgQualities; + std::unordered_map orgSizes; + std::unordered_map orgQualityCounts; + collectOrgInfo(maxQuality, orgQualities, orgSizes, orgQualityCounts, + validators); + + // Generate a config + Config cfg = getTestConfig(); + cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true; + cfg.generateQuorumSetForTesting(validators); + cfg.NODE_SEED = sks.front(); + + // Create an application + VirtualClock clock; + Application::pointer app = createTestApplication(clock, cfg); + + for_versions_from( + static_cast( + APPLICATION_SPECIFIC_NOMINATION_LEADER_ELECTION_PROTOCOL_VERSION), + *app, [&]() { + // Run for `numLedgers` slots, recording the number of times each + // node wins nomination + UnorderedMap publishCounts; + HerderImpl& herder = dynamic_cast(app->getHerder()); + SCP& scp = herder.getSCP(); + int fastTimeouts = 0; + for (int i = 0; i < numLedgers; ++i) + { + auto s = std::make_shared(i, scp); + TestNominationProtocol np(*s); + + std::set const& leaders = + np.updateRoundLeadersForTesting(); + REQUIRE(leaders.size() == 1); + for (NodeID const& leader : leaders) + { + ++publishCounts[leader]; + } + + if (np.fastTimedOut()) + { + ++fastTimeouts; + } + } + + CLOG_INFO(Herder, "Fast Timeouts: {} ({}%)", fastTimeouts, + fastTimeouts * 100.0 / numLedgers); + + // Compute total expected normalized weight across all nodes + double totalNormalizedWeight = 0.0; + for (ValidatorEntry const& validator : validators) + { + totalNormalizedWeight += expectedNormalizedWeight( + orgQualityCounts, maxQuality, + orgQualities.at(validator.mHomeDomain), + orgSizes.at(validator.mHomeDomain)); + } + + // Check validator win rates + std::map orgPublishCounts; + for (ValidatorEntry const& validator : validators) + { + NodeID const& nodeID = validator.mKey; + int publishCount = publishCounts[nodeID]; + + // Compute and report node's win rate + double winRate = static_cast(publishCount) / numLedgers; + CLOG_INFO(Herder, "Node {} win rate: {} (published {} ledgers)", + cfg.toShortString(nodeID), winRate, publishCount); + + // Expected win rate is `weight / total weight` + double expectedWinRate = + expectedNormalizedWeight( + orgQualityCounts, maxQuality, + orgQualities.at(validator.mHomeDomain), + orgSizes.at(validator.mHomeDomain)) / + totalNormalizedWeight; + + // Check that actual win rate is within .05 of expected win + // rate. + REQUIRE_THAT(winRate, + Catch::Matchers::WithinAbs(expectedWinRate, 0.05)); + + // Record org publish counts for the next set of checks + orgPublishCounts[validator.mHomeDomain] += publishCount; + } + + // Check org win rates + for (auto const& [org, count] : orgPublishCounts) + { + // Compute and report org's win rate + double winRate = static_cast(count) / numLedgers; + CLOG_INFO(Herder, "Org {} win rate: {} (published {} ledgers)", + org, winRate, count); + + // Expected win rate is `weight / total weight` + double expectedWinRate = + expectedOrgNormalizedWeight(orgQualityCounts, maxQuality, + orgQualities.at(org)) / + totalNormalizedWeight; + + // Check that actual win rate is within .05 of expected win + // rate. + REQUIRE_THAT(winRate, + Catch::Matchers::WithinAbs(expectedWinRate, 0.05)); + } + }); +} + +// Test that the nomination algorithm produces a fair distribution of ledger +// publishers. +TEST_CASE_VERSIONS("Fair nomination win rates", "[herder]") +{ + SECTION("3 tier 1 validators, 1 org") + { + auto [sks, validators] = simpleThreeNode(); + testWinProbabilities(sks, validators, 10000); + } + + SECTION("11 tier 1 validators, 3 unbalanced orgs") + { + auto [sks, validators] = unbalancedOrgs(); + testWinProbabilities(sks, validators, 10000); + } + + SECTION("Tier 1-like topology") + { + auto [sks, validators] = teir1Like(); + testWinProbabilities(sks, validators, 10000); + } + + SECTION("Random topology") + { + for (int i = 0; i < 10; ++i) + { + auto [sks, validators] = randomTopology(50); + testWinProbabilities(sks, validators, 10000); + } + } +} \ No newline at end of file diff --git a/src/main/Config.cpp b/src/main/Config.cpp index cecfa2ddf5..ca1e316b1f 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -306,6 +306,8 @@ Config::Config() : NODE_SEED(SecretKey::random()) EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = false; EMIT_LEDGER_CLOSE_META_EXT_V1 = false; + FORCE_OLD_STYLE_LEADER_ELECTION = false; + #ifdef BUILD_TESTS TEST_CASES_ENABLED = false; #endif @@ -572,7 +574,7 @@ Config::toString(ValidatorQuality q) const return kQualities[static_cast(q)]; } -Config::ValidatorQuality +ValidatorQuality Config::parseQuality(std::string const& q) const { auto it = std::find(kQualities.begin(), kQualities.end(), q); @@ -581,7 +583,7 @@ Config::parseQuality(std::string const& q) const if (it != kQualities.end()) { - res = static_cast( + res = static_cast( std::distance(kQualities.begin(), it)); } else @@ -592,7 +594,7 @@ Config::parseQuality(std::string const& q) const return res; } -std::vector +std::vector Config::parseValidators( std::shared_ptr validators, UnorderedMap const& domainQualityMap) @@ -715,7 +717,7 @@ Config::parseValidators( return res; } -UnorderedMap +UnorderedMap Config::parseDomainsQuality(std::shared_ptr domainsQuality) { UnorderedMap res; @@ -1616,6 +1618,10 @@ Config::processConfig(std::shared_ptr t) { it->second(); } + else if (item.first == "FORCE_OLD_STYLE_LEADER_ELECTION") + { + FORCE_OLD_STYLE_LEADER_ELECTION = readBool(item); + } else { std::string err("Unknown configuration entry: '"); @@ -1806,6 +1812,7 @@ Config::processConfig(std::shared_ptr t) LOG_INFO(DEFAULT_LOG, "Generated QUORUM_SET: {}", autoQSetStr); QUORUM_SET = autoQSet; verifyHistoryValidatorsBlocking(validators); + setValidatorWeightConfig(validators); // count the number of domains UnorderedSet domains; for (auto const& v : validators) @@ -2428,5 +2435,74 @@ Config::toString(SCPQuorumSet const& qset) return fw.write(json); } +void +Config::setValidatorWeightConfig(std::vector const& validators) +{ + releaseAssert(!VALIDATOR_WEIGHT_CONFIG.has_value()); + + if (!NODE_IS_VALIDATOR) + { + // There is no reason to populate VALIDATOR_WEIGHT_CONFIG if the node is + // not a validator. + return; + } + + ValidatorWeightConfig& vwc = VALIDATOR_WEIGHT_CONFIG.emplace(); + ValidatorQuality highestQuality = ValidatorQuality::VALIDATOR_LOW_QUALITY; + ValidatorQuality lowestQuality = + ValidatorQuality::VALIDATOR_CRITICAL_QUALITY; + UnorderedMap> + homeDomainsByQuality; + for (auto const& v : validators) + { + if (!vwc.mValidatorEntries.try_emplace(v.mKey, v).second) + { + throw std::invalid_argument( + fmt::format(FMT_STRING("Duplicate validator entry for '{}'"), + KeyUtils::toStrKey(v.mKey))); + } + ++vwc.mHomeDomainSizes[v.mHomeDomain]; + highestQuality = std::max(highestQuality, v.mQuality); + lowestQuality = std::min(lowestQuality, v.mQuality); + homeDomainsByQuality[v.mQuality].insert(v.mHomeDomain); + } + + // Highest quality level has weight UINT64_MAX + vwc.mQualityWeights[highestQuality] = UINT64_MAX; + + // Assign weights to the remaining quality levels + for (int q = static_cast(highestQuality) - 1; + q >= static_cast(lowestQuality); --q) + { + // Next higher quality level + ValidatorQuality higherQuality = static_cast(q + 1); + + // Get weight of next higher quality level + uint64 higherWeight = vwc.mQualityWeights.at(higherQuality); + + // Get number of orgs at next higher quality level. Add 1 for the + // virtual org containing this quality level. + uint64 higherOrgs = homeDomainsByQuality[higherQuality].size() + 1; + + // The weight of this quality level is the higher quality weight divided + // by the number of orgs at that quality level multiplied by 10 + vwc.mQualityWeights[static_cast(q)] = + higherWeight / (higherOrgs * 10); + } + + // Special case: LOW quality level has weight 0 + vwc.mQualityWeights[ValidatorQuality::VALIDATOR_LOW_QUALITY] = 0; +} + +#ifdef BUILD_TESTS +void +Config::generateQuorumSetForTesting( + std::vector const& validators) +{ + QUORUM_SET = generateQuorumSet(validators); + setValidatorWeightConfig(validators); +} +#endif // BUILD_TESTS + std::string const Config::STDIN_SPECIAL_NAME = "stdin"; } diff --git a/src/main/Config.h b/src/main/Config.h index 45e344f205..600a349259 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -37,24 +37,39 @@ enum class ValidationThresholdLevels : int ALL_REQUIRED = 2 }; -class Config : public std::enable_shared_from_this +enum class ValidatorQuality : int { - enum class ValidatorQuality : int - { - VALIDATOR_LOW_QUALITY = 0, - VALIDATOR_MED_QUALITY = 1, - VALIDATOR_HIGH_QUALITY = 2, - VALIDATOR_CRITICAL_QUALITY = 3 - }; + VALIDATOR_LOW_QUALITY = 0, + VALIDATOR_MED_QUALITY = 1, + VALIDATOR_HIGH_QUALITY = 2, + VALIDATOR_CRITICAL_QUALITY = 3 +}; - struct ValidatorEntry - { - std::string mName; - std::string mHomeDomain; - ValidatorQuality mQuality; - PublicKey mKey; - bool mHasHistory; - }; +struct ValidatorEntry +{ + std::string mName; + std::string mHomeDomain; + ValidatorQuality mQuality; + PublicKey mKey; + bool mHasHistory; +}; + +// This struct holds information necessary to compute the weight of a validator +// for leader election +struct ValidatorWeightConfig +{ + // Mapping from node ids to info about each validator + UnorderedMap mValidatorEntries; + + // Mapping from org names to the number of validators in that org + UnorderedMap mHomeDomainSizes; + + // Weights for each quality level + UnorderedMap mQualityWeights; +}; + +class Config : public std::enable_shared_from_this +{ void validateConfig(ValidationThresholdLevels thresholdLevel); void loadQset(std::shared_ptr group, SCPQuorumSet& qset, @@ -111,6 +126,11 @@ class Config : public std::enable_shared_from_this std::string const& valuesName, std::string const& distributionName); + // Sets VALIDATOR_WEIGHT_CONFIG based on the content of `validators`. No-op + // if this node is not a validator. + void + setValidatorWeightConfig(std::vector const& validators); + public: static const uint32 CURRENT_LEDGER_PROTOCOL_VERSION; @@ -602,6 +622,15 @@ class Config : public std::enable_shared_from_this std::map VALIDATOR_NAMES; + // Information necessary to compute the weight of a validator for leader + // election. Nullopt if this node is not a validator, or if this node is + // using manual quorum set configuration. + std::optional VALIDATOR_WEIGHT_CONFIG; + + // Revert to the old, application-agnostic nomination weight function for + // SCP leader election. + bool FORCE_OLD_STYLE_LEADER_ELECTION; + // History config std::map HISTORY; @@ -668,6 +697,11 @@ class Config : public std::enable_shared_from_this // case. This is used right now in the signal handler to exit() instead of // doing a graceful shutdown bool TEST_CASES_ENABLED; + + // Set QUORUM_SET using automatic quorum set configuration based on + // `validators`. + void + generateQuorumSetForTesting(std::vector const& validators); #endif #ifdef BEST_OFFER_DEBUGGING diff --git a/src/main/test/ConfigTests.cpp b/src/main/test/ConfigTests.cpp index e6154825d6..29938731a9 100644 --- a/src/main/test/ConfigTests.cpp +++ b/src/main/test/ConfigTests.cpp @@ -216,6 +216,67 @@ TEST_CASE("load validators config", "[config]") REQUIRE(c.KNOWN_PEERS.size() == 13); REQUIRE(c.PREFERRED_PEERS.size() == 2); // 2 other "domainA" validators REQUIRE(c.HISTORY.size() == 20); + + // Check that VALIDATOR_WEIGHT_CONFIG is correctly loaded + SECTION("VALIDATOR_WEIGHT_CONFIG") + { + REQUIRE(c.VALIDATOR_WEIGHT_CONFIG.has_value()); + ValidatorWeightConfig const& vwc = c.VALIDATOR_WEIGHT_CONFIG.value(); + + // Should be 30 validators, counting 'self' + REQUIRE(vwc.mValidatorEntries.size() == 30); + + // Check a validator with a home domain defined in [[HOME_DOMAINS]] + NodeID const e2 = KeyUtils::fromStrKey( + "GCBEPQHP3D42OHQMA54NRF3E4BAJ6T7NZP7Q7URI2VWNQDJPXTDA3SBJ"); + ValidatorEntry const& e2Entry = vwc.mValidatorEntries.at(e2); + REQUIRE(e2Entry.mName == "e2"); + REQUIRE(e2Entry.mHomeDomain == "domainE"); + REQUIRE(e2Entry.mQuality == ValidatorQuality::VALIDATOR_LOW_QUALITY); + REQUIRE(e2Entry.mKey == e2); + REQUIRE(!e2Entry.mHasHistory); + + // Check a validator with a home domain not defined in [[HOME_DOMAINS]] + NodeID const d1 = KeyUtils::fromStrKey( + "GDD3QN464732BXOZ7UZ43I5KR76X5YPNCUZMUCI4HJXRYQL4EJR6QAZL"); + ValidatorEntry const& d1Entry = vwc.mValidatorEntries.at(d1); + REQUIRE(d1Entry.mName == "d1"); + REQUIRE(d1Entry.mHomeDomain == "domainD"); + REQUIRE(d1Entry.mQuality == ValidatorQuality::VALIDATOR_MED_QUALITY); + REQUIRE(d1Entry.mKey == d1); + REQUIRE(!d1Entry.mHasHistory); + + // Check self + NodeID const self = c.NODE_SEED.getPublicKey(); + ValidatorEntry const& selfEntry = vwc.mValidatorEntries.at(self); + REQUIRE(selfEntry.mName == "self"); + REQUIRE(selfEntry.mHomeDomain == "domainA"); + REQUIRE(selfEntry.mQuality == ValidatorQuality::VALIDATOR_HIGH_QUALITY); + REQUIRE(selfEntry.mKey == self); + REQUIRE(!selfEntry.mHasHistory); + + // Check home-domain count for each domain + UnorderedMap const expectedHomeDomainSizes = { + {"domainA", 3}, {"domainB", 3}, {"domainC", 2}, {"domainD", 2}, + {"domainE", 3}, {"domainF", 1}, {"domainG", 1}, {"domainH", 1}, + {"domainI", 1}, {"domainJ", 1}, {"domainK", 3}, {"domainL", 3}, + {"domainM", 3}, {"domainN", 3}}; + REQUIRE(vwc.mHomeDomainSizes == expectedHomeDomainSizes); + + // Check quality weights + UnorderedMap const expectedQualityWeights = { + {ValidatorQuality::VALIDATOR_LOW_QUALITY, 0}, + // Denominator is 1600 because there are 3 HIGH orgs + 1 virtual org + // containing the MED orgs. This means the quality level should be + // HIGH_QUALITY_WEIGHT / (4 * 10), or + // UINT64_MAX / 40 / (4 * 10), which is UINT64_MAX / 1600 + {ValidatorQuality::VALIDATOR_MED_QUALITY, UINT64_MAX / 1600}, + // Denominator is 40 because there are 3 CRITICAL orgs + 1 virtual + // org containing the HIGH quality orgs + {ValidatorQuality::VALIDATOR_HIGH_QUALITY, UINT64_MAX / 40}, + {ValidatorQuality::VALIDATOR_CRITICAL_QUALITY, UINT64_MAX}}; + REQUIRE(vwc.mQualityWeights == expectedQualityWeights); + } } TEST_CASE("bad validators configs", "[config]") diff --git a/src/scp/LocalNode.cpp b/src/scp/LocalNode.cpp index 5f8139424b..62c908d96b 100644 --- a/src/scp/LocalNode.cpp +++ b/src/scp/LocalNode.cpp @@ -89,50 +89,6 @@ LocalNode::forAllNodes(SCPQuorumSet const& qset, return true; } -uint64 -LocalNode::computeWeight(uint64 m, uint64 total, uint64 threshold) -{ - uint64 res; - releaseAssert(threshold <= total); - // Since threshold <= total, calculating res=m*threshold/total will always - // produce res <= m, and we do not need to handle the possibility of this - // call returning false (indicating overflow). - bool noOverflow = bigDivideUnsigned(res, m, threshold, total, ROUND_UP); - releaseAssert(noOverflow); - return res; -} - -// if a validator is repeated multiple times its weight is only the -// weight of the first occurrence -uint64 -LocalNode::getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset) -{ - uint64 n = qset.threshold; - uint64 d = qset.innerSets.size() + qset.validators.size(); - uint64 res; - - for (auto const& qsetNode : qset.validators) - { - if (qsetNode == nodeID) - { - res = computeWeight(UINT64_MAX, d, n); - return res; - } - } - - for (auto const& q : qset.innerSets) - { - uint64 leafW = getNodeWeight(nodeID, q); - if (leafW) - { - res = computeWeight(leafW, d, n); - return res; - } - } - - return 0; -} - bool LocalNode::isQuorumSliceInternal(SCPQuorumSet const& qset, std::vector const& nodeSet) diff --git a/src/scp/LocalNode.h b/src/scp/LocalNode.h index 04f208c0dd..6611494b61 100644 --- a/src/scp/LocalNode.h +++ b/src/scp/LocalNode.h @@ -50,10 +50,6 @@ class LocalNode static bool forAllNodes(SCPQuorumSet const& qset, std::function proc); - // returns the weight of the node within the qset - // normalized between 0-UINT64_MAX - static uint64 getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset); - // Tests this node against nodeSet for the specified qSethash. static bool isQuorumSlice(SCPQuorumSet const& qSet, std::vector const& nodeSet); @@ -108,8 +104,6 @@ class LocalNode std::string to_string(SCPQuorumSet const& qSet) const; - static uint64 computeWeight(uint64 m, uint64 total, uint64 threshold); - protected: // returns a quorum set {{ nodeID }} static SCPQuorumSet buildSingletonQSet(NodeID const& nodeID); diff --git a/src/scp/NominationProtocol.cpp b/src/scp/NominationProtocol.cpp index 6b44566538..39df387941 100644 --- a/src/scp/NominationProtocol.cpp +++ b/src/scp/NominationProtocol.cpp @@ -254,6 +254,15 @@ NominationProtocol::updateRoundLeaders() } return true; }); + + if (topPriority == 0) + { + // No one had priority, so all nodes would choose themselves + // resulting in a timeout. Clear newRoundLeaders, allowing the + // algorithm to fast timeout and try again. + newRoundLeaders.clear(); + } + // expand mRoundLeaders with the newly computed leaders auto oldSize = mRoundLeaders.size(); mRoundLeaders.insert(newRoundLeaders.begin(), newRoundLeaders.end()); @@ -306,17 +315,8 @@ NominationProtocol::getNodePriority(NodeID const& nodeID, { ZoneScoped; uint64 res; - uint64 w; - - if (nodeID == mSlot.getLocalNode()->getNodeID()) - { - // local node is in all quorum sets - w = UINT64_MAX; - } - else - { - w = LocalNode::getNodeWeight(nodeID, qset); - } + uint64 w = mSlot.getSCPDriver().getNodeWeight( + nodeID, qset, nodeID == mSlot.getLocalNode()->getNodeID()); // if w > 0; w is inclusive here as // 0 <= hashNode <= UINT64_MAX diff --git a/src/scp/SCPDriver.cpp b/src/scp/SCPDriver.cpp index 5973d132c3..ca56cb06f4 100644 --- a/src/scp/SCPDriver.cpp +++ b/src/scp/SCPDriver.cpp @@ -10,11 +10,28 @@ #include "crypto/KeyUtils.h" #include "crypto/SecretKey.h" #include "util/GlobalChecks.h" +#include "util/numeric.h" #include "xdrpp/marshal.h" namespace stellar { +namespace +{ +uint64 +computeWeight(uint64 m, uint64 total, uint64 threshold) +{ + uint64 res; + releaseAssert(threshold <= total); + // Since threshold <= total, calculating res=m*threshold/total will always + // produce res <= m, and we do not need to handle the possibility of this + // call returning false (indicating overflow). + bool noOverflow = bigDivideUnsigned(res, m, threshold, total, ROUND_UP); + releaseAssert(noOverflow); + return res; +} +} // namespace + bool WrappedValuePtrComparator::operator()(ValueWrapperPtr const& l, ValueWrapperPtr const& r) const @@ -145,4 +162,43 @@ SCPDriver::computeTimeout(uint32 roundNumber) } return std::chrono::seconds(timeoutInSeconds); } + +// if a validator is repeated multiple times its weight is only the +// weight of the first occurrence +uint64 +SCPDriver::getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset, + bool isLocalNode) const +{ + if (isLocalNode) + { + // local node is in all quorum sets + return UINT64_MAX; + } + + uint64 n = qset.threshold; + uint64 d = qset.innerSets.size() + qset.validators.size(); + uint64 res; + + for (auto const& qsetNode : qset.validators) + { + if (qsetNode == nodeID) + { + res = computeWeight(UINT64_MAX, d, n); + return res; + } + } + + for (auto const& q : qset.innerSets) + { + uint64 leafW = SCPDriver::getNodeWeight(nodeID, q, isLocalNode); + if (leafW) + { + res = computeWeight(leafW, d, n); + return res; + } + } + + return 0; +} + } diff --git a/src/scp/SCPDriver.h b/src/scp/SCPDriver.h index c1f9085c63..b630ebb7dd 100644 --- a/src/scp/SCPDriver.h +++ b/src/scp/SCPDriver.h @@ -179,6 +179,12 @@ class SCPDriver // quorum can exchange 4 messages virtual std::chrono::milliseconds computeTimeout(uint32 roundNumber); + // returns the weight of the node within the qset normalized between + // 0-UINT64_MAX. If `nodeID` is the local node, then set `isLocalNode` to + // `true`. + virtual uint64 getNodeWeight(NodeID const& nodeID, SCPQuorumSet const& qset, + bool isLocalNode) const; + // Inform about events happening within the consensus algorithm. // `valueExternalized` is called at most once per slot when the slot diff --git a/src/scp/test/SCPUnitTests.cpp b/src/scp/test/SCPUnitTests.cpp index 72b6e7ccf4..a6066b89c3 100644 --- a/src/scp/test/SCPUnitTests.cpp +++ b/src/scp/test/SCPUnitTests.cpp @@ -15,40 +15,6 @@ isNear(uint64 r, double target) return (std::abs(v - target) < .01); } -TEST_CASE("nomination weight", "[scp]") -{ - SIMULATION_CREATE_NODE(0); - SIMULATION_CREATE_NODE(1); - SIMULATION_CREATE_NODE(2); - SIMULATION_CREATE_NODE(3); - SIMULATION_CREATE_NODE(4); - SIMULATION_CREATE_NODE(5); - - SCPQuorumSet qSet; - qSet.threshold = 3; - qSet.validators.push_back(v0NodeID); - qSet.validators.push_back(v1NodeID); - qSet.validators.push_back(v2NodeID); - qSet.validators.push_back(v3NodeID); - - uint64 result = LocalNode::getNodeWeight(v2NodeID, qSet); - - REQUIRE(isNear(result, .75)); - - result = LocalNode::getNodeWeight(v4NodeID, qSet); - REQUIRE(result == 0); - - SCPQuorumSet iQSet; - iQSet.threshold = 1; - iQSet.validators.push_back(v4NodeID); - iQSet.validators.push_back(v5NodeID); - qSet.innerSets.push_back(iQSet); - - result = LocalNode::getNodeWeight(v4NodeID, qSet); - - REQUIRE(isNear(result, .6 * .5)); -} - class TestNominationSCP : public SCPDriver { public: @@ -135,6 +101,42 @@ class TestNominationSCP : public SCPDriver } }; +TEST_CASE("nomination weight", "[scp]") +{ + SIMULATION_CREATE_NODE(0); + SIMULATION_CREATE_NODE(1); + SIMULATION_CREATE_NODE(2); + SIMULATION_CREATE_NODE(3); + SIMULATION_CREATE_NODE(4); + SIMULATION_CREATE_NODE(5); + + SCPQuorumSet qSet; + qSet.threshold = 3; + qSet.validators.push_back(v0NodeID); + qSet.validators.push_back(v1NodeID); + qSet.validators.push_back(v2NodeID); + qSet.validators.push_back(v3NodeID); + + TestNominationSCP const nomSCP(v0NodeID, qSet); + + uint64 result = nomSCP.getNodeWeight(v2NodeID, qSet, false); + + REQUIRE(isNear(result, .75)); + + result = nomSCP.getNodeWeight(v4NodeID, qSet, false); + REQUIRE(result == 0); + + SCPQuorumSet iQSet; + iQSet.threshold = 1; + iQSet.validators.push_back(v4NodeID); + iQSet.validators.push_back(v5NodeID); + qSet.innerSets.push_back(iQSet); + + result = nomSCP.getNodeWeight(v4NodeID, qSet, false); + + REQUIRE(isNear(result, .6 * .5)); +} + class NominationTestHandler : public NominationProtocol { public: