diff --git a/Cargo.lock b/Cargo.lock index c963a3f5eef1..f5c9b8d0694f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "binary-merkle-tree" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "hash-db", "log", @@ -1847,18 +1847,32 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16" +checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" dependencies = [ "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", "fiat-crypto", - "packed_simd_2", "platforms", + "rustc_version", "subtle", "zeroize", ] +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "cxx" version = "1.0.102" @@ -2339,6 +2353,16 @@ dependencies = [ "signature 1.6.4", ] +[[package]] +name = "ed25519" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.1.0", +] + [[package]] name = "ed25519-dalek" version = "1.0.1" @@ -2346,13 +2370,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ "curve25519-dalek 3.2.0", - "ed25519", + "ed25519 1.5.3", "rand 0.7.3", "serde", "sha2 0.9.9", "zeroize", ] +[[package]] +name = "ed25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +dependencies = [ + "curve25519-dalek 4.0.0", + "ed25519 2.2.2", + "serde", + "sha2 0.10.7", + "zeroize", +] + [[package]] name = "ed25519-zebra" version = "3.1.0" @@ -2831,7 +2868,7 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "fork-tree" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", ] @@ -2854,7 +2891,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "frame-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-support-procedural", @@ -2879,7 +2916,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "Inflector", "array-bytes", @@ -2927,7 +2964,7 @@ dependencies = [ [[package]] name = "frame-election-provider-solution-type" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2938,7 +2975,7 @@ dependencies = [ [[package]] name = "frame-election-provider-support" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-election-provider-solution-type", "frame-support", @@ -2955,7 +2992,7 @@ dependencies = [ [[package]] name = "frame-executive" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -2984,7 +3021,7 @@ dependencies = [ [[package]] name = "frame-remote-externalities" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-recursion", "futures", @@ -3006,7 +3043,7 @@ dependencies = [ [[package]] name = "frame-support" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "aquamarine", "bitflags 1.3.2", @@ -3044,7 +3081,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "Inflector", "cfg-expr", @@ -3062,7 +3099,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", @@ -3074,7 +3111,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro2", "quote", @@ -3084,7 +3121,7 @@ dependencies = [ [[package]] name = "frame-support-test" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-executive", @@ -3111,7 +3148,7 @@ dependencies = [ [[package]] name = "frame-support-test-pallet" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -3124,7 +3161,7 @@ dependencies = [ [[package]] name = "frame-system" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "cfg-if", "frame-support", @@ -3143,7 +3180,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -3158,7 +3195,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "sp-api", @@ -3167,7 +3204,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "parity-scale-codec", @@ -3342,7 +3379,7 @@ dependencies = [ [[package]] name = "generate-bags" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "chrono", "frame-election-provider-support", @@ -4472,12 +4509,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "libm" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" - [[package]] name = "libp2p" version = "0.51.3" @@ -4607,7 +4638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e2d584751cecb2aabaa56106be6be91338a60a0f4e420cf2af639204f596fc1" dependencies = [ "bs58", - "ed25519-dalek", + "ed25519-dalek 1.0.1", "log", "multiaddr", "multihash", @@ -5346,7 +5377,7 @@ dependencies = [ [[package]] name = "mmr-gadget" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "log", @@ -5365,7 +5396,7 @@ dependencies = [ [[package]] name = "mmr-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "anyhow", "jsonrpsee", @@ -5889,20 +5920,10 @@ dependencies = [ "sha2 0.10.7", ] -[[package]] -name = "packed_simd_2" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" -dependencies = [ - "cfg-if", - "libm", -] - [[package]] name = "pallet-assets" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -5917,7 +5938,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -5933,7 +5954,7 @@ dependencies = [ [[package]] name = "pallet-authorship" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -5947,7 +5968,7 @@ dependencies = [ [[package]] name = "pallet-babe" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -5971,7 +5992,7 @@ dependencies = [ [[package]] name = "pallet-bags-list" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "aquamarine", "docify", @@ -5993,7 +6014,7 @@ dependencies = [ [[package]] name = "pallet-bags-list-remote-tests" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-election-provider-support", "frame-remote-externalities", @@ -6012,7 +6033,7 @@ dependencies = [ [[package]] name = "pallet-balances" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6027,7 +6048,7 @@ dependencies = [ [[package]] name = "pallet-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -6046,7 +6067,7 @@ dependencies = [ [[package]] name = "pallet-beefy-mmr" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "binary-merkle-tree", @@ -6070,7 +6091,7 @@ dependencies = [ [[package]] name = "pallet-bounties" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6088,7 +6109,7 @@ dependencies = [ [[package]] name = "pallet-child-bounties" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6107,7 +6128,7 @@ dependencies = [ [[package]] name = "pallet-collective" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6124,7 +6145,7 @@ dependencies = [ [[package]] name = "pallet-conviction-voting" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "assert_matches", "frame-benchmarking", @@ -6141,7 +6162,7 @@ dependencies = [ [[package]] name = "pallet-democracy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6159,7 +6180,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-multi-phase" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -6182,7 +6203,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-support-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -6195,7 +6216,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" version = "5.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6214,7 +6235,7 @@ dependencies = [ [[package]] name = "pallet-fast-unstake" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "docify", "frame-benchmarking", @@ -6233,7 +6254,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6256,7 +6277,7 @@ dependencies = [ [[package]] name = "pallet-identity" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "enumflags2", "frame-benchmarking", @@ -6272,7 +6293,7 @@ dependencies = [ [[package]] name = "pallet-im-online" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6292,7 +6313,7 @@ dependencies = [ [[package]] name = "pallet-indices" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6309,7 +6330,7 @@ dependencies = [ [[package]] name = "pallet-membership" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6326,7 +6347,7 @@ dependencies = [ [[package]] name = "pallet-message-queue" version = "7.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6345,7 +6366,7 @@ dependencies = [ [[package]] name = "pallet-mmr" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6362,7 +6383,7 @@ dependencies = [ [[package]] name = "pallet-multisig" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6378,7 +6399,7 @@ dependencies = [ [[package]] name = "pallet-nis" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6394,7 +6415,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools" version = "1.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -6413,7 +6434,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools-benchmarking" version = "1.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -6433,7 +6454,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools-runtime-api" version = "1.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "pallet-nomination-pools", "parity-scale-codec", @@ -6444,7 +6465,7 @@ dependencies = [ [[package]] name = "pallet-offences" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -6461,7 +6482,7 @@ dependencies = [ [[package]] name = "pallet-offences-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -6485,7 +6506,7 @@ dependencies = [ [[package]] name = "pallet-preimage" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6502,7 +6523,7 @@ dependencies = [ [[package]] name = "pallet-proxy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6517,7 +6538,7 @@ dependencies = [ [[package]] name = "pallet-ranked-collective" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6535,7 +6556,7 @@ dependencies = [ [[package]] name = "pallet-recovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6550,7 +6571,7 @@ dependencies = [ [[package]] name = "pallet-referenda" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "assert_matches", "frame-benchmarking", @@ -6569,7 +6590,7 @@ dependencies = [ [[package]] name = "pallet-salary" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6587,7 +6608,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "docify", "frame-benchmarking", @@ -6605,7 +6626,7 @@ dependencies = [ [[package]] name = "pallet-session" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -6626,7 +6647,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6642,7 +6663,7 @@ dependencies = [ [[package]] name = "pallet-society" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6660,7 +6681,7 @@ dependencies = [ [[package]] name = "pallet-staking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -6683,7 +6704,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -6694,7 +6715,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-fn" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "log", "sp-arithmetic", @@ -6703,7 +6724,7 @@ dependencies = [ [[package]] name = "pallet-staking-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "sp-api", @@ -6712,7 +6733,7 @@ dependencies = [ [[package]] name = "pallet-state-trie-migration" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6729,7 +6750,7 @@ dependencies = [ [[package]] name = "pallet-sudo" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6744,7 +6765,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6762,7 +6783,7 @@ dependencies = [ [[package]] name = "pallet-tips" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6781,7 +6802,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-support", "frame-system", @@ -6797,7 +6818,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "jsonrpsee", "pallet-transaction-payment-rpc-runtime-api", @@ -6813,7 +6834,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "pallet-transaction-payment", "parity-scale-codec", @@ -6825,7 +6846,7 @@ dependencies = [ [[package]] name = "pallet-treasury" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6842,7 +6863,7 @@ dependencies = [ [[package]] name = "pallet-uniques" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6857,7 +6878,7 @@ dependencies = [ [[package]] name = "pallet-utility" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6873,7 +6894,7 @@ dependencies = [ [[package]] name = "pallet-vesting" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -6888,7 +6909,7 @@ dependencies = [ [[package]] name = "pallet-whitelist" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-benchmarking", "frame-support", @@ -7272,9 +7293,11 @@ name = "polkadot-approval-distribution" version = "0.9.43" dependencies = [ "assert_matches", + "bitvec", "env_logger 0.9.3", "futures", "futures-timer", + "itertools", "log", "polkadot-node-jaeger", "polkadot-node-metrics", @@ -7581,10 +7604,13 @@ dependencies = [ "async-trait", "bitvec", "derive_more", + "env_logger 0.9.3", "futures", "futures-timer", + "itertools", "kvdb", "kvdb-memorydb", + "log", "lru 0.11.0", "merlin 2.0.1", "parity-scale-codec", @@ -8053,6 +8079,7 @@ dependencies = [ name = "polkadot-node-primitives" version = "0.9.43" dependencies = [ + "bitvec", "bounded-vec", "futures", "parity-scale-codec", @@ -8103,6 +8130,7 @@ name = "polkadot-node-subsystem-types" version = "0.9.43" dependencies = [ "async-trait", + "bitvec", "derive_more", "futures", "orchestra", @@ -10084,7 +10112,7 @@ dependencies = [ [[package]] name = "sc-allocator" version = "4.1.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "log", "sp-core", @@ -10095,7 +10123,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -10123,7 +10151,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "futures-timer", @@ -10146,7 +10174,7 @@ dependencies = [ [[package]] name = "sc-block-builder" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -10161,7 +10189,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "memmap2", "sc-chain-spec-derive", @@ -10180,7 +10208,7 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -10191,7 +10219,7 @@ dependencies = [ [[package]] name = "sc-cli" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "chrono", @@ -10230,7 +10258,7 @@ dependencies = [ [[package]] name = "sc-client-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "fnv", "futures", @@ -10256,7 +10284,7 @@ dependencies = [ [[package]] name = "sc-client-db" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "hash-db", "kvdb", @@ -10282,7 +10310,7 @@ dependencies = [ [[package]] name = "sc-consensus" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -10307,7 +10335,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "fork-tree", @@ -10343,7 +10371,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "jsonrpsee", @@ -10365,7 +10393,7 @@ dependencies = [ [[package]] name = "sc-consensus-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "async-channel", @@ -10399,7 +10427,7 @@ dependencies = [ [[package]] name = "sc-consensus-beefy-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "jsonrpsee", @@ -10418,7 +10446,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "fork-tree", "parity-scale-codec", @@ -10431,7 +10459,7 @@ dependencies = [ [[package]] name = "sc-consensus-grandpa" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ahash 0.8.3", "array-bytes", @@ -10472,7 +10500,7 @@ dependencies = [ [[package]] name = "sc-consensus-grandpa-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "finality-grandpa", "futures", @@ -10492,7 +10520,7 @@ dependencies = [ [[package]] name = "sc-consensus-slots" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -10515,7 +10543,7 @@ dependencies = [ [[package]] name = "sc-executor" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "parking_lot 0.12.1", @@ -10537,7 +10565,7 @@ dependencies = [ [[package]] name = "sc-executor-common" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "sc-allocator", "sp-maybe-compressed-blob", @@ -10549,7 +10577,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "anyhow", "cfg-if", @@ -10566,7 +10594,7 @@ dependencies = [ [[package]] name = "sc-informant" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ansi_term", "futures", @@ -10582,7 +10610,7 @@ dependencies = [ [[package]] name = "sc-keystore" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "parking_lot 0.12.1", @@ -10596,7 +10624,7 @@ dependencies = [ [[package]] name = "sc-network" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "async-channel", @@ -10637,7 +10665,7 @@ dependencies = [ [[package]] name = "sc-network-bitswap" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-channel", "cid", @@ -10657,7 +10685,7 @@ dependencies = [ [[package]] name = "sc-network-common" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "bitflags 1.3.2", @@ -10674,7 +10702,7 @@ dependencies = [ [[package]] name = "sc-network-gossip" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ahash 0.8.3", "futures", @@ -10692,7 +10720,7 @@ dependencies = [ [[package]] name = "sc-network-light" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "async-channel", @@ -10713,7 +10741,7 @@ dependencies = [ [[package]] name = "sc-network-sync" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "async-channel", @@ -10747,7 +10775,7 @@ dependencies = [ [[package]] name = "sc-network-transactions" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "futures", @@ -10765,7 +10793,7 @@ dependencies = [ [[package]] name = "sc-offchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "bytes", @@ -10799,7 +10827,7 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "log", "substrate-prometheus-endpoint", @@ -10808,7 +10836,7 @@ dependencies = [ [[package]] name = "sc-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "jsonrpsee", @@ -10839,7 +10867,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "jsonrpsee", "parity-scale-codec", @@ -10858,7 +10886,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "http", "jsonrpsee", @@ -10873,7 +10901,7 @@ dependencies = [ [[package]] name = "sc-rpc-spec-v2" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "futures", @@ -10901,7 +10929,7 @@ dependencies = [ [[package]] name = "sc-service" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "directories", @@ -10965,7 +10993,7 @@ dependencies = [ [[package]] name = "sc-state-db" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "log", "parity-scale-codec", @@ -10976,7 +11004,7 @@ dependencies = [ [[package]] name = "sc-storage-monitor" version = "0.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "clap 4.3.19", "fs4", @@ -10990,7 +11018,7 @@ dependencies = [ [[package]] name = "sc-sync-state-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "jsonrpsee", "parity-scale-codec", @@ -11009,7 +11037,7 @@ dependencies = [ [[package]] name = "sc-sysinfo" version = "6.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "libc", @@ -11028,7 +11056,7 @@ dependencies = [ [[package]] name = "sc-telemetry" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "chrono", "futures", @@ -11047,7 +11075,7 @@ dependencies = [ [[package]] name = "sc-tracing" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ansi_term", "atty", @@ -11076,7 +11104,7 @@ dependencies = [ [[package]] name = "sc-tracing-proc-macro" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -11087,7 +11115,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -11113,7 +11141,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -11129,7 +11157,7 @@ dependencies = [ [[package]] name = "sc-utils" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-channel", "futures", @@ -11658,14 +11686,14 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "snow" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733" +checksum = "0c9d1425eb528a21de2755c75af4c9b5d57f50a0d4c3b7f1828a4cd03f8ba155" dependencies = [ "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-rc.1", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "ring 0.16.20", "rustc_version", @@ -11713,7 +11741,7 @@ dependencies = [ [[package]] name = "sp-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "hash-db", "log", @@ -11734,7 +11762,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "Inflector", "blake2", @@ -11748,7 +11776,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" version = "23.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -11761,7 +11789,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" version = "16.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "integer-sqrt", "num-traits", @@ -11775,7 +11803,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -11788,7 +11816,7 @@ dependencies = [ [[package]] name = "sp-block-builder" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "sp-api", "sp-inherents", @@ -11799,7 +11827,7 @@ dependencies = [ [[package]] name = "sp-blockchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "log", @@ -11817,7 +11845,7 @@ dependencies = [ [[package]] name = "sp-consensus" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "futures", @@ -11832,7 +11860,7 @@ dependencies = [ [[package]] name = "sp-consensus-aura" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "parity-scale-codec", @@ -11849,7 +11877,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "parity-scale-codec", @@ -11868,7 +11896,7 @@ dependencies = [ [[package]] name = "sp-consensus-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "lazy_static", "parity-scale-codec", @@ -11887,7 +11915,7 @@ dependencies = [ [[package]] name = "sp-consensus-grandpa" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "finality-grandpa", "log", @@ -11905,7 +11933,7 @@ dependencies = [ [[package]] name = "sp-consensus-slots" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -11917,7 +11945,7 @@ dependencies = [ [[package]] name = "sp-core" version = "21.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "arrayvec 0.7.4", @@ -11964,7 +11992,7 @@ dependencies = [ [[package]] name = "sp-core-hashing" version = "9.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "blake2b_simd", "byteorder", @@ -11977,7 +12005,7 @@ dependencies = [ [[package]] name = "sp-core-hashing-proc-macro" version = "9.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "quote", "sp-core-hashing", @@ -11987,7 +12015,7 @@ dependencies = [ [[package]] name = "sp-database" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "kvdb", "parking_lot 0.12.1", @@ -11996,7 +12024,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "8.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro2", "quote", @@ -12006,7 +12034,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.19.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "environmental", "parity-scale-codec", @@ -12017,7 +12045,7 @@ dependencies = [ [[package]] name = "sp-genesis-builder" version = "0.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "serde_json", "sp-api", @@ -12028,7 +12056,7 @@ dependencies = [ [[package]] name = "sp-inherents" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "impl-trait-for-tuples", @@ -12042,11 +12070,10 @@ dependencies = [ [[package]] name = "sp-io" version = "23.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "bytes", - "ed25519", - "ed25519-dalek", + "ed25519-dalek 2.0.0", "libsecp256k1", "log", "parity-scale-codec", @@ -12067,7 +12094,7 @@ dependencies = [ [[package]] name = "sp-keyring" version = "24.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "lazy_static", "sp-core", @@ -12078,7 +12105,7 @@ dependencies = [ [[package]] name = "sp-keystore" version = "0.27.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "parking_lot 0.12.1", @@ -12090,7 +12117,7 @@ dependencies = [ [[package]] name = "sp-maybe-compressed-blob" version = "4.1.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "thiserror", "zstd 0.12.4", @@ -12099,7 +12126,7 @@ dependencies = [ [[package]] name = "sp-metadata-ir" version = "0.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-metadata", "parity-scale-codec", @@ -12110,7 +12137,7 @@ dependencies = [ [[package]] name = "sp-mmr-primitives" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ckb-merkle-mountain-range", "log", @@ -12128,7 +12155,7 @@ dependencies = [ [[package]] name = "sp-npos-elections" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -12142,7 +12169,7 @@ dependencies = [ [[package]] name = "sp-offchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "sp-api", "sp-core", @@ -12152,7 +12179,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" version = "8.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "backtrace", "lazy_static", @@ -12162,7 +12189,7 @@ dependencies = [ [[package]] name = "sp-rpc" version = "6.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "rustc-hash", "serde", @@ -12172,7 +12199,7 @@ dependencies = [ [[package]] name = "sp-runtime" version = "24.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "either", "hash256-std-hasher", @@ -12194,7 +12221,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "17.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "bytes", "impl-trait-for-tuples", @@ -12212,7 +12239,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "11.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "Inflector", "proc-macro-crate", @@ -12224,7 +12251,7 @@ dependencies = [ [[package]] name = "sp-session" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -12239,7 +12266,7 @@ dependencies = [ [[package]] name = "sp-staking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -12253,7 +12280,7 @@ dependencies = [ [[package]] name = "sp-state-machine" version = "0.28.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "hash-db", "log", @@ -12274,11 +12301,11 @@ dependencies = [ [[package]] name = "sp-statement-store" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "aes-gcm 0.10.2", - "curve25519-dalek 3.2.0", - "ed25519-dalek", + "curve25519-dalek 4.0.0", + "ed25519-dalek 2.0.0", "hkdf", "parity-scale-codec", "rand 0.8.5", @@ -12292,18 +12319,18 @@ dependencies = [ "sp-runtime-interface", "sp-std", "thiserror", - "x25519-dalek 2.0.0-pre.1", + "x25519-dalek 2.0.0", ] [[package]] name = "sp-std" version = "8.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" [[package]] name = "sp-storage" version = "13.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "impl-serde", "parity-scale-codec", @@ -12316,7 +12343,7 @@ dependencies = [ [[package]] name = "sp-timestamp" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "parity-scale-codec", @@ -12329,7 +12356,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "10.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "sp-std", @@ -12341,7 +12368,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "sp-api", "sp-runtime", @@ -12350,7 +12377,7 @@ dependencies = [ [[package]] name = "sp-transaction-storage-proof" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "parity-scale-codec", @@ -12365,7 +12392,7 @@ dependencies = [ [[package]] name = "sp-trie" version = "22.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ahash 0.8.3", "hash-db", @@ -12388,7 +12415,7 @@ dependencies = [ [[package]] name = "sp-version" version = "22.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "impl-serde", "parity-scale-codec", @@ -12405,7 +12432,7 @@ dependencies = [ [[package]] name = "sp-version-proc-macro" version = "8.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "proc-macro2", @@ -12416,7 +12443,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "14.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "anyhow", "impl-trait-for-tuples", @@ -12429,7 +12456,7 @@ dependencies = [ [[package]] name = "sp-weights" version = "20.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "parity-scale-codec", "scale-info", @@ -12670,12 +12697,12 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" [[package]] name = "substrate-frame-rpc-system" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "frame-system-rpc-runtime-api", "futures", @@ -12694,7 +12721,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "hyper", "log", @@ -12706,7 +12733,7 @@ dependencies = [ [[package]] name = "substrate-rpc-client" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "jsonrpsee", @@ -12719,7 +12746,7 @@ dependencies = [ [[package]] name = "substrate-state-trie-migration-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "jsonrpsee", "parity-scale-codec", @@ -12736,7 +12763,7 @@ dependencies = [ [[package]] name = "substrate-test-client" version = "2.0.1" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "array-bytes", "async-trait", @@ -12762,7 +12789,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "futures", "substrate-test-utils-derive", @@ -12772,7 +12799,7 @@ dependencies = [ [[package]] name = "substrate-test-utils-derive" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -12783,7 +12810,7 @@ dependencies = [ [[package]] name = "substrate-wasm-builder" version = "5.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "ansi_term", "build-helper", @@ -13691,7 +13718,7 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "try-runtime-cli" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#94be94be6d26becd2395b58ae09ca31f596afe7d" +source = "git+https://github.com/paritytech/substrate?branch=master#51695bb7009ea2e0996eb94ab4dfdc643a076702" dependencies = [ "async-trait", "clap 4.3.19", @@ -14513,7 +14540,7 @@ dependencies = [ "tokio", "webpki 0.21.4", "webrtc-util", - "x25519-dalek 2.0.0-pre.1", + "x25519-dalek 2.0.0", "x509-parser 0.13.2", ] @@ -15032,12 +15059,13 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0-pre.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5da623d8af10a62342bcbbb230e33e58a63255a58012f8653c578e54bab48df" +checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ - "curve25519-dalek 3.2.0", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", + "serde", "zeroize", ] diff --git a/node/core/approval-voting/Cargo.toml b/node/core/approval-voting/Cargo.toml index f31c7d0a1694..8748eaa0bec4 100644 --- a/node/core/approval-voting/Cargo.toml +++ b/node/core/approval-voting/Cargo.toml @@ -17,6 +17,7 @@ schnorrkel = "0.9.1" kvdb = "0.13.0" derive_more = "0.99.17" thiserror = "1.0.31" +itertools = "0.10.5" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } @@ -30,6 +31,9 @@ sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "mast sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = ["full_crypto"] } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +# Needed for migration test helpers +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +rand_core = "0.5.1" [dev-dependencies] async-trait = "0.1.57" @@ -43,3 +47,5 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } assert_matches = "1.4.0" kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +log = "0.4.17" +env_logger = "0.9.0" diff --git a/node/core/approval-voting/src/approval_checking.rs b/node/core/approval-voting/src/approval_checking.rs index f345b57029b5..5d24ff164193 100644 --- a/node/core/approval-voting/src/approval_checking.rs +++ b/node/core/approval-voting/src/approval_checking.rs @@ -17,7 +17,7 @@ //! Utilities for checking whether a candidate has been approved under a given block. use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use polkadot_primitives::ValidatorIndex; use crate::{ @@ -472,9 +472,9 @@ mod tests { } .into(); - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: BitVec::default(), + assigned_validators: BitVec::default(), our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -509,22 +509,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -581,22 +581,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -647,9 +647,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 5], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -691,9 +691,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -731,9 +731,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -776,9 +776,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -843,9 +843,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -934,9 +934,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1041,15 +1041,15 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1094,12 +1094,12 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, our_approval_sig: None, - assignments: bitvec![u8, BitOrderLsb0; 0; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3], approved: false, } .into(); diff --git a/node/core/approval-voting/src/approval_db/mod.rs b/node/core/approval-voting/src/approval_db/mod.rs index f0ace95613da..20fb6aa82d8d 100644 --- a/node/core/approval-voting/src/approval_db/mod.rs +++ b/node/core/approval-voting/src/approval_db/mod.rs @@ -31,3 +31,4 @@ //! time being we share the same DB with the rest of Substrate. pub mod v1; +pub mod v2; diff --git a/node/core/approval-voting/src/approval_db/v1/mod.rs b/node/core/approval-voting/src/approval_db/v1/mod.rs index c31389269d2e..011d0a559c02 100644 --- a/node/core/approval-voting/src/approval_db/v1/mod.rs +++ b/node/core/approval-voting/src/approval_db/v1/mod.rs @@ -23,144 +23,15 @@ //! require a db migration (check `node/service/src/parachains_db/upgrade.rs`). use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche}; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; -use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_node_primitives::approval::v1::{AssignmentCert, DelayTranche}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; +use std::collections::BTreeMap; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; - -use crate::{ - backend::{Backend, BackendWriteOp}, - persisted_entries, -}; - -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; - -#[cfg(test)] -pub mod tests; - -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - -// slot_duration * 2 + DelayTranche gives the number of delay tranches since the -// unix epoch. -#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] -pub struct Tick(u64); - -/// Convenience type definition -pub type Bitfield = BitVec; - -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} +use super::v2::Bitfield; /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -171,15 +42,7 @@ pub struct OurAssignment { // Whether the assignment has been triggered already. pub triggered: bool, } - -/// Metadata regarding a specific tranche of assignments for a specific candidate. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct TrancheEntry { - pub tranche: DelayTranche, - // Assigned validators, and the instant we received their assignment, rounded - // to the nearest tick. - pub assignments: Vec<(ValidatorIndex, Tick)>, -} +use super::v2::TrancheEntry; /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -226,126 +89,3 @@ pub struct BlockEntry { pub approved_bitfield: Bitfield, pub children: Vec, } - -impl From for Tick { - fn from(tick: crate::Tick) -> Tick { - Tick(tick) - } -} - -impl From for crate::Tick { - fn from(tick: Tick) -> crate::Tick { - tick.0 - } -} - -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} diff --git a/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs new file mode 100644 index 000000000000..3d1f28ee1937 --- /dev/null +++ b/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -0,0 +1,241 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::{StoredBlockRange, *}; +use crate::backend::Backend; +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; +use polkadot_node_subsystem_util::database::Database; +use std::{collections::HashSet, sync::Arc}; + +use ::test_helpers::dummy_candidate_receipt; + +fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + +fn make_block_entry_v1( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v1::BlockEntry { + crate::approval_db::v1::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +/// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v1(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v1(&candidate_hash) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_v2_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v1_to_v2_fill_test_data( + db: Arc, + config: Config, +) -> Result> { + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_receipt(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v1( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v1::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v1::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v1::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + assignments: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v1(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v1(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v1( + candidate_entry: crate::approval_db::v1::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v1( + block_entry: crate::approval_db::v1::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/node/core/approval-voting/src/approval_db/v2/mod.rs b/node/core/approval-voting/src/approval_db/v2/mod.rs new file mode 100644 index 000000000000..66df6ee8f653 --- /dev/null +++ b/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -0,0 +1,394 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 2 of the DB schema. + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, + ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use std::{collections::BTreeMap, sync::Arc}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend}, + persisted_entries, +}; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +pub mod migration_helpers; +#[cfg(test)] +pub mod tests; + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(Into::into)) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); + +// slot_duration * 2 + DelayTranche gives the number of delay tranches since the +// unix epoch. +#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] +pub struct Tick(u64); + +/// Convenience type definition +pub type Bitfield = BitVec; + +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// Details pertaining to our assignment on a block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurAssignment { + /// Our assignment certificate. + pub cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + pub tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + pub validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + pub triggered: bool, +} + +/// Metadata regarding a specific tranche of assignments for a specific candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct TrancheEntry { + pub tranche: DelayTranche, + // Assigned validators, and the instant we received their assignment, rounded + // to the nearest tick. + pub assignments: Vec<(ValidatorIndex, Tick)>, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +impl From for Tick { + fn from(tick: crate::Tick) -> Tick { + Tick(tick) + } +} + +impl From for crate::Tick { + fn from(tick: Tick) -> crate::Tick { + tick.0 + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in v1 format. +pub fn load_candidate_entry_v1( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v1 format. +pub fn load_block_entry_v1( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/node/core/approval-voting/src/approval_db/v1/tests.rs b/node/core/approval-voting/src/approval_db/v2/tests.rs similarity index 99% rename from node/core/approval-voting/src/approval_db/v1/tests.rs rename to node/core/approval-voting/src/approval_db/v2/tests.rs index 07d8242b772e..50a5a924ca8d 100644 --- a/node/core/approval-voting/src/approval_db/v1/tests.rs +++ b/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -40,10 +40,6 @@ fn make_db() -> (DbBackend, Arc) { (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_block_entry( block_hash: Hash, parent_hash: Hash, @@ -60,9 +56,14 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), + distributed_assignments: Default::default(), } } +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); @@ -96,7 +97,7 @@ fn read_write() { backing_group: GroupIndex(1), our_assignment: None, our_approval_sig: None, - assignments: Default::default(), + assigned_validators: Default::default(), approved: false, }, )] diff --git a/node/core/approval-voting/src/backend.rs b/node/core/approval-voting/src/backend.rs index 87d67c52c467..374e7a826d19 100644 --- a/node/core/approval-voting/src/backend.rs +++ b/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; use std::collections::HashMap; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -44,6 +44,7 @@ pub enum BackendWriteOp { } /// An abstraction over backend storage for the logic of this subsystem. +/// Implementation must always target latest storage version. pub trait Backend { /// Load a block entry from the DB. fn load_block_entry(&self, hash: &Hash) -> SubsystemResult>; @@ -52,6 +53,7 @@ pub trait Backend { &self, candidate_hash: &CandidateHash, ) -> SubsystemResult>; + /// Load all blocks at a specific height. fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult>; /// Load all block from the DB. @@ -64,6 +66,18 @@ pub trait Backend { I: IntoIterator; } +/// A read only backed to enable db migration from version 1 of DB. +pub trait V1ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/node/core/approval-voting/src/criteria.rs b/node/core/approval-voting/src/criteria.rs index 0e1d18198c21..1f751e2bf140 100644 --- a/node/core/approval-voting/src/criteria.rs +++ b/node/core/approval-voting/src/criteria.rs @@ -18,7 +18,9 @@ use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ - self as approval_types, AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory, + self as approval_types, + v1::{AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory}, + v2::{AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, VrfOutput, VrfProof, VrfSignature}, }; use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, @@ -30,6 +32,7 @@ use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; +use itertools::Itertools; use std::collections::{hash_map::Entry, HashMap}; use super::LOG_TARGET; @@ -37,7 +40,7 @@ use super::LOG_TARGET; /// Details pertaining to our assignment on a block. #[derive(Debug, Clone, Encode, Decode, PartialEq)] pub struct OurAssignment { - cert: AssignmentCert, + cert: AssignmentCertV2, tranche: DelayTranche, validator_index: ValidatorIndex, // Whether the assignment has been triggered already. @@ -45,7 +48,7 @@ pub struct OurAssignment { } impl OurAssignment { - pub(crate) fn cert(&self) -> &AssignmentCert { + pub(crate) fn cert(&self) -> &AssignmentCertV2 { &self.cert } @@ -66,8 +69,8 @@ impl OurAssignment { } } -impl From for OurAssignment { - fn from(entry: crate::approval_db::v1::OurAssignment) -> Self { +impl From for OurAssignment { + fn from(entry: crate::approval_db::v2::OurAssignment) -> Self { OurAssignment { cert: entry.cert, tranche: entry.tranche, @@ -77,7 +80,7 @@ impl From for OurAssignment { } } -impl From for crate::approval_db::v1::OurAssignment { +impl From for crate::approval_db::v2::OurAssignment { fn from(entry: OurAssignment) -> Self { Self { cert: entry.cert, @@ -88,17 +91,97 @@ impl From for crate::approval_db::v1::OurAssignment { } } -fn relay_vrf_modulo_transcript(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { - // combine the relay VRF story with a sample number. - let mut t = Transcript::new(approval_types::RELAY_VRF_MODULO_CONTEXT); - t.append_message(b"RC-VRF", &relay_vrf_story.0); - sample.using_encoded(|s| t.append_message(b"sample", s)); +// Combines the relay VRF story with a sample number if any. +fn relay_vrf_modulo_transcript_inner( + mut transcript: Transcript, + relay_vrf_story: RelayVRFStory, + sample: Option, +) -> Transcript { + transcript.append_message(b"RC-VRF", &relay_vrf_story.0); - t + if let Some(sample) = sample { + sample.using_encoded(|s| transcript.append_message(b"sample", s)); + } + + transcript +} + +fn relay_vrf_modulo_transcript_v1(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v1::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + Some(sample), + ) +} + +fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v2::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + None, + ) +} + +/// A hard upper bound on num_cores * target_checkers / num_validators +const MAX_MODULO_SAMPLES: usize = 40; + +use std::convert::AsMut; + +fn clone_into_array(slice: &[T]) -> A +where + A: Default + AsMut<[T]>, + T: Clone, +{ + let mut a = A::default(); + >::as_mut(&mut a).clone_from_slice(slice); + a +} + +struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); + +impl Default for BigArray { + fn default() -> Self { + BigArray([0u8; MAX_MODULO_SAMPLES * 4]) + } +} + +impl AsMut<[u8]> for BigArray { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +/// Takes the VRF output as input and returns a Vec of cores the validator is assigned +/// to as a tranche0 checker. +fn relay_vrf_modulo_cores( + vrf_in_out: &VRFInOut, + // Configuration - `relay_vrf_modulo_samples`. + num_samples: u32, + // Configuration - `n_cores`. + max_cores: u32, +) -> Vec { + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + vrf_in_out + .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) + .0 + .chunks_exact(4) + .take(num_samples as usize) + .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) + .unique() + .collect::>() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::CORE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::CORE_RANDOMNESS_CONTEXT); // interpret as little-endian u32. let random_core = u32::from_le_bytes(bytes) % n_cores; @@ -106,7 +189,7 @@ fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { } fn relay_vrf_delay_transcript(relay_vrf_story: RelayVRFStory, core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::RELAY_VRF_DELAY_CONTEXT); + let mut t = Transcript::new(approval_types::v1::RELAY_VRF_DELAY_CONTEXT); t.append_message(b"RC-VRF", &relay_vrf_story.0); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t @@ -117,7 +200,7 @@ fn relay_vrf_delay_tranche( num_delay_tranches: u32, zeroth_delay_tranche_width: u32, ) -> DelayTranche { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::TRANCHE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::TRANCHE_RANDOMNESS_CONTEXT); // interpret as little-endian u32 and reduce by the number of tranches. let wide_tranche = @@ -128,13 +211,13 @@ fn relay_vrf_delay_tranche( } fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::ASSIGNED_CORE_CONTEXT); + let mut t = Transcript::new(approval_types::v1::ASSIGNED_CORE_CONTEXT); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t } /// Information about the world assignments are being produced in. -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct Config { /// The assignment public keys for validators. assignment_keys: Vec, @@ -175,12 +258,13 @@ pub(crate) trait AssignmentCriteria { fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + // Backing groups for each "leaving core". + backing_groups: Vec, ) -> Result; } @@ -194,25 +278,25 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { check_assignment_cert( - claimed_core_index, + claimed_core_bitfield, validator_index, config, relay_vrf_story, assignment, - backing_group, + backing_groups, ) } } @@ -233,6 +317,7 @@ pub(crate) fn compute_assignments( relay_vrf_story: RelayVRFStory, config: &Config, leaving_cores: impl IntoIterator + Clone, + enable_v2_assignments: bool, ) -> HashMap { if config.n_cores == 0 || config.assignment_keys.is_empty() || @@ -291,14 +376,25 @@ pub(crate) fn compute_assignments( let mut assignments = HashMap::new(); // First run `RelayVRFModulo` for each sample. - compute_relay_vrf_modulo_assignments( - &assignments_key, - index, - config, - relay_vrf_story.clone(), - leaving_cores.iter().cloned(), - &mut assignments, - ); + if enable_v2_assignments { + compute_relay_vrf_modulo_assignments_v2( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } else { + compute_relay_vrf_modulo_assignments_v1( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } // Then run `RelayVRFDelay` once for the whole block. compute_relay_vrf_delay_assignments( @@ -313,7 +409,7 @@ pub(crate) fn compute_assignments( assignments } -fn compute_relay_vrf_modulo_assignments( +fn compute_relay_vrf_modulo_assignments_v1( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, config: &Config, @@ -329,7 +425,7 @@ fn compute_relay_vrf_modulo_assignments( // into closure. let core = &mut core; assignments_key.vrf_sign_extra_after_check( - relay_vrf_modulo_transcript(relay_vrf_story.clone(), rvm_sample), + relay_vrf_modulo_transcript_v1(relay_vrf_story.clone(), rvm_sample), |vrf_in_out| { *core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); if let Some((candidate_hash, _)) = @@ -357,15 +453,15 @@ fn compute_relay_vrf_modulo_assignments( // has been executed. let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; // All assignments of type RelayVRFModulo have tranche 0. assignments.entry(core).or_insert(OurAssignment { - cert, + cert: cert.into(), tranche: 0, validator_index, triggered: false, @@ -374,6 +470,84 @@ fn compute_relay_vrf_modulo_assignments( } } +fn assigned_cores_transcript(core_bitfield: &CoreBitfield) -> Transcript { + let mut t = Transcript::new(approval_types::v2::ASSIGNED_CORE_CONTEXT); + core_bitfield.using_encoded(|s| t.append_message(b"cores", s)); + t +} + +fn compute_relay_vrf_modulo_assignments_v2( + assignments_key: &schnorrkel::Keypair, + validator_index: ValidatorIndex, + config: &Config, + relay_vrf_story: RelayVRFStory, + leaving_cores: Vec<(CandidateHash, CoreIndex)>, + assignments: &mut HashMap, +) { + let mut assigned_cores = Vec::new(); + let leaving_cores = leaving_cores.iter().map(|(_, core)| core).collect::>(); + + let maybe_assignment = { + let assigned_cores = &mut assigned_cores; + assignments_key.vrf_sign_extra_after_check( + relay_vrf_modulo_transcript_v2(relay_vrf_story.clone()), + |vrf_in_out| { + *assigned_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ) + .into_iter() + .filter(|core| leaving_cores.contains(&core)) + .collect::>(); + + if !assigned_cores.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?assigned_cores, + ?validator_index, + tranche = 0, + "RelayVRFModuloCompact Assignment." + ); + + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + Some(assigned_cores_transcript(&assignment_bitfield)) + } else { + None + } + }, + ) + }; + + if let Some(assignment) = maybe_assignment.map(|(vrf_in_out, vrf_proof, _)| { + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: assignment_bitfield.clone(), + }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), + }, + }; + + // All assignments of type RelayVRFModulo have tranche 0. + OurAssignment { cert, tranche: 0, validator_index, triggered: false } + }) { + for core_index in assigned_cores { + assignments.insert(core_index, assignment.clone()); + } + } +} + fn compute_relay_vrf_delay_assignments( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, @@ -392,11 +566,11 @@ fn compute_relay_vrf_delay_assignments( config.zeroth_delay_tranche_width, ); - let cert = AssignmentCert { - kind: AssignmentCertKind::RelayVRFDelay { core_index: core }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFDelay { core_index: core }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; @@ -453,12 +627,15 @@ pub(crate) enum InvalidAssignmentReason { VRFModuloOutputMismatch, VRFDelayCoreIndexMismatch, VRFDelayOutputMismatch, + InvalidArguments, + /// Assignment vrf check resulted in 0 assigned cores. + NullAssignment, } /// Checks the crypto of an assignment cert. Failure conditions: /// * Validator index out of bounds /// * VRF signature check fails -/// * VRF output doesn't match assigned core +/// * VRF output doesn't match assigned cores /// * Core is not covered by extra data in signature /// * Core index out of bounds /// * Sample is out of bounds @@ -467,12 +644,12 @@ pub(crate) enum InvalidAssignmentReason { /// This function does not check whether the core is actually a valid assignment or not. That should /// be done outside the scope of this function. pub(crate) fn check_assignment_cert( - claimed_core_index: CoreIndex, + claimed_core_indices: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { use InvalidAssignmentReason as Reason; @@ -484,52 +661,133 @@ pub(crate) fn check_assignment_cert( let public = schnorrkel::PublicKey::from_bytes(validator_public.as_slice()) .map_err(|_| InvalidAssignment(Reason::InvalidAssignmentKey))?; - if claimed_core_index.0 >= config.n_cores { - return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + // Check that we have all backing groups for claimed cores. + if claimed_core_indices.count_ones() == 0 || + claimed_core_indices.count_ones() != backing_groups.len() + { + return Err(InvalidAssignment(Reason::InvalidArguments)) } // Check that the validator was not part of the backing group // and not already assigned. - let is_in_backing = - is_in_backing_group(&config.validator_groups, validator_index, backing_group); + for (claimed_core, backing_group) in claimed_core_indices.iter_ones().zip(backing_groups.iter()) + { + if claimed_core >= config.n_cores as usize { + return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + } - if is_in_backing { - return Err(InvalidAssignment(Reason::IsInBackingGroup)) + let is_in_backing = + is_in_backing_group(&config.validator_groups, validator_index, *backing_group); + + if is_in_backing { + return Err(InvalidAssignment(Reason::IsInBackingGroup)) + } } - let vrf_signature = &assignment.vrf; - match assignment.kind { - AssignmentCertKind::RelayVRFModulo { sample } => { - if sample >= config.relay_vrf_modulo_samples { + let vrf_output = &assignment.vrf.output; + let vrf_proof = &assignment.vrf.proof; + let first_claimed_core_index = + claimed_core_indices.first_one().expect("Checked above; qed") as u32; + + match &assignment.kind { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => { + // Check that claimed core bitfield match the one from certificate. + if &claimed_core_indices != core_bitfield { + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + + let (vrf_in_out, _) = public + .vrf_verify_extra( + relay_vrf_modulo_transcript_v2(relay_vrf_story), + &vrf_output.0, + &vrf_proof.0, + assigned_cores_transcript(core_bitfield), + ) + .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + + let resulting_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ); + + // Currently validators can opt out of checking specific cores. + // This is the same issue to how validator can opt out and not send their assignments in + // the first place. Ensure that the `vrf_in_out` actually includes all of the claimed + // cores. + for claimed_core_index in claimed_core_indices.iter_ones() { + if !resulting_cores.contains(&CoreIndex(claimed_core_index as u32)) { + gum::debug!( + target: LOG_TARGET, + ?resulting_cores, + ?claimed_core_indices, + vrf_modulo_cores = ?resulting_cores, + "Assignment claimed cores mismatch", + ); + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + } + + Ok(0) + }, + AssignmentCertKindV2::RelayVRFModulo { sample } => { + if *sample >= config.relay_vrf_modulo_samples { return Err(InvalidAssignment(Reason::SampleOutOfBounds)) } + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::warn!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFModulo` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + let (vrf_in_out, _) = public .vrf_verify_extra( - relay_vrf_modulo_transcript(relay_vrf_story, sample), - &vrf_signature.output.0, - &vrf_signature.proof.0, - assigned_core_transcript(claimed_core_index), + relay_vrf_modulo_transcript_v1(relay_vrf_story, *sample), + &vrf_output.0, + &vrf_proof.0, + assigned_core_transcript(CoreIndex(first_claimed_core_index)), ) .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + let core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); // ensure that the `vrf_in_out` actually gives us the claimed core. - if relay_vrf_modulo_core(&vrf_in_out, config.n_cores) == claimed_core_index { + if core.0 == first_claimed_core_index { Ok(0) } else { + gum::debug!( + target: LOG_TARGET, + ?core, + ?claimed_core_indices, + "Assignment claimed cores mismatch", + ); Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) } }, - AssignmentCertKind::RelayVRFDelay { core_index } => { - if core_index != claimed_core_index { + AssignmentCertKindV2::RelayVRFDelay { core_index } => { + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::debug!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFDelay` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + + if core_index.0 != first_claimed_core_index { return Err(InvalidAssignment(Reason::VRFDelayCoreIndexMismatch)) } let (vrf_in_out, _) = public .vrf_verify( - relay_vrf_delay_transcript(relay_vrf_story, core_index), - &vrf_signature.output.0, - &vrf_signature.proof.0, + relay_vrf_delay_transcript(relay_vrf_story, *core_index), + &vrf_output.0, + &vrf_proof.0, ) .map_err(|_| InvalidAssignment(Reason::VRFDelayOutputMismatch))?; @@ -550,6 +808,19 @@ fn is_in_backing_group( validator_groups.get(group).map_or(false, |g| g.contains(&validator)) } +/// Migration helpers. +impl From for OurAssignment { + fn from(value: crate::approval_db::v1::OurAssignment) -> Self { + Self { + cert: value.cert.into(), + tranche: value.tranche, + validator_index: value.validator_index, + // Whether the assignment has been triggered already. + triggered: value.triggered, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -630,10 +901,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(1)), (c_b, CoreIndex(1), GroupIndex(0))], + false, ); // Note that alice is in group 0, which was the backing group for core 1. @@ -665,10 +937,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(0)), (c_b, CoreIndex(1), GroupIndex(1))], + false, ); assert_eq!(assignments.len(), 1); @@ -692,19 +965,21 @@ mod tests { validator_groups: Default::default(), n_cores: 0, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![], + false, ); assert!(assignments.is_empty()); } + #[derive(Debug)] struct MutatedAssignment { - core: CoreIndex, - cert: AssignmentCert, - group: GroupIndex, + cores: CoreBitfield, + cert: AssignmentCertV2, + groups: Vec, own_group: GroupIndex, val_index: ValidatorIndex, config: Config, @@ -729,12 +1004,12 @@ mod tests { validator_groups: basic_groups(n_validators, n_cores), n_cores: n_cores as u32, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 15, n_delay_tranches: 40, }; let relay_vrf_story = RelayVRFStory([42u8; 32]); - let assignments = compute_assignments( + let mut assignments = compute_assignments( &keystore, relay_vrf_story.clone(), &config, @@ -747,19 +1022,42 @@ mod tests { ) }) .collect::>(), + false, ); + // Extend with v2 assignments as well + assignments.extend(compute_assignments( + &keystore, + relay_vrf_story.clone(), + &config, + (0..n_cores) + .map(|i| { + ( + CandidateHash(Hash::repeat_byte(i as u8)), + CoreIndex(i as u32), + group_for_core(i), + ) + }) + .collect::>(), + true, + )); + let mut counted = 0; for (core, assignment) in assignments { + let cores = match assignment.cert.kind.clone() { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => core_bitfield, + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => core.into(), + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.into(), + }; + let mut mutated = MutatedAssignment { - core, - group: group_for_core(core.0 as _), + cores: cores.clone(), + groups: cores.iter_ones().map(|core| group_for_core(core)).collect(), cert: assignment.cert, own_group: GroupIndex(0), val_index: ValidatorIndex(0), config: config.clone(), }; - let expected = match f(&mut mutated) { None => continue, Some(e) => e, @@ -768,16 +1066,16 @@ mod tests { counted += 1; let is_good = check_assignment_cert( - mutated.core, + mutated.cores, mutated.val_index, &mutated.config, relay_vrf_story.clone(), &mutated.cert, - mutated.group, + mutated.groups, ) .is_ok(); - assert_eq!(expected, is_good) + assert_eq!(expected, is_good); } assert!(counted > 0); @@ -791,7 +1089,7 @@ mod tests { #[test] fn check_rejects_claimed_core_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { - m.core.0 += 100; + m.cores = CoreIndex(100).into(); Some(false) }); } @@ -799,7 +1097,7 @@ mod tests { #[test] fn check_rejects_in_backing_group() { check_mutated_assignments(200, 100, 25, |m| { - m.group = m.own_group; + m.groups[0] = m.own_group; Some(false) }); } @@ -815,9 +1113,10 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { check_mutated_assignments(40, 10, 8, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -828,9 +1127,14 @@ mod tests { #[test] fn check_rejects_modulo_bad_vrf() { check_mutated_assignments(200, 100, 25, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFModulo { .. } => { + m.cert.vrf = vrf_signature; + Some(false) + }, + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -842,10 +1146,11 @@ mod tests { fn check_rejects_modulo_sample_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { sample } => { + AssignmentCertKindV2::RelayVRFModulo { sample } => { m.config.relay_vrf_modulo_samples = sample; Some(false) }, + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield: _ } => Some(true), _ => None, // skip everything else. } }); @@ -855,8 +1160,11 @@ mod tests { fn check_rejects_delay_claimed_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + // for core in &mut m.cores { + // core.0 = (core.0 + 1) % 100; + // } + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); Some(false) }, _ => None, // skip everything else. @@ -868,8 +1176,10 @@ mod tests { fn check_rejects_modulo_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFModulo { .. } | + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); + Some(false) }, _ => None, // skip everything else. diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index c504ba71b3c2..4345380ed2f9 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -30,7 +30,10 @@ use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ - approval::{self as approval_types, BlockApprovalMeta, RelayVRFStory}, + approval::{ + self as approval_types, + v1::{BlockApprovalMeta, RelayVRFStory}, + }, MAX_FINALITY_LAG, }; use polkadot_node_subsystem::{ @@ -53,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v1; +use super::approval_db::v2; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -92,7 +95,7 @@ enum ImportedBlockInfoError { FutureCancelled(&'static str, futures::channel::oneshot::Canceled), #[error(transparent)] - ApprovalError(approval_types::ApprovalError), + ApprovalError(approval_types::v1::ApprovalError), #[error("block is already finalized")] BlockAlreadyFinalized, @@ -216,7 +219,7 @@ async fn imported_block_info( .ok_or(ImportedBlockInfoError::SessionInfoUnavailable)?; let (assignments, slot, relay_vrf_story) = { - let unsafe_vrf = approval_types::babe_unsafe_vrf_info(&block_header); + let unsafe_vrf = approval_types::v1::babe_unsafe_vrf_info(&block_header); match unsafe_vrf { Some(unsafe_vrf) => { @@ -497,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v1::BlockEntry { + let block_entry = v2::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -510,6 +513,7 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + distributed_assignments: Default::default(), }; gum::trace!( @@ -588,11 +592,11 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v1::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ - approval::{VrfSignature, VrfTranscript}, + approval::v1::{VrfSignature, VrfTranscript}, DISPUTE_WINDOW, }; use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; @@ -608,7 +612,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v1::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -654,7 +658,7 @@ pub(crate) mod tests { fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -667,13 +671,14 @@ pub(crate) mod tests { fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, _validator_index: polkadot_primitives::ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result + { Ok(0) } } @@ -1252,7 +1257,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v1::BlockEntry { + v2::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1262,6 +1267,7 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + distributed_assignments: Default::default(), } .into(), ); @@ -1294,7 +1300,7 @@ pub(crate) mod tests { // the first candidate should be insta-approved // the second should not let entry: BlockEntry = - v1::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) .unwrap() .unwrap() .into(); diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index b29e47b4c435..beaca43ee528 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -25,7 +25,11 @@ use jaeger::{hash_to_trace_identifier, PerLeafSpan}; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{BlockApprovalMeta, DelayTranche, IndirectSignedApprovalVote}, + v2::{ + AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, + IndirectAssignmentCertV2, + }, }, ValidationResult, DISPUTE_WINDOW, }; @@ -75,12 +79,13 @@ use std::{ }; use approval_checking::RequiredTranches; +use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; mod approval_checking; -mod approval_db; +pub mod approval_db; mod backend; mod criteria; mod import; @@ -89,8 +94,9 @@ mod persisted_entries; mod time; use crate::{ - approval_db::v1::{Config as DatabaseConfig, DbBackend}, + approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, + criteria::InvalidAssignmentReason, }; #[cfg(test)] @@ -109,7 +115,7 @@ const APPROVAL_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(1024) { const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; -const LOG_TARGET: &str = "parachain::approval-voting"; +pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] @@ -379,8 +385,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v1::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v1::DbBackend::new(self.db.clone(), config); + approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; @@ -741,14 +747,15 @@ enum Action { tick: Tick, }, LaunchApproval { + claimed_candidate_indices: CandidateBitfield, candidate_hash: CandidateHash, - indirect_cert: IndirectAssignmentCert, + indirect_cert: IndirectAssignmentCertV2, assignment_tranche: DelayTranche, relay_block_hash: Hash, - candidate_index: CandidateIndex, session: SessionIndex, candidate: CandidateReceipt, backing_group: GroupIndex, + distribute_assignment: bool, }, NoteApprovedInChainSelection(Hash), IssueApproval(CandidateHash, ApprovalVoteRequest), @@ -963,14 +970,15 @@ async fn handle_actions( actions_iter = next_actions.into_iter(); }, Action::LaunchApproval { + claimed_candidate_indices, candidate_hash, indirect_cert, assignment_tranche, relay_block_hash, - candidate_index, session, candidate, backing_group, + distribute_assignment, } => { // Don't launch approval work if the node is syncing. if let Mode::Syncing(_) = *mode { @@ -991,10 +999,12 @@ async fn handle_actions( launch_approval_span.add_string_tag("block-hash", format!("{:?}", block_hash)); let validator_index = indirect_cert.validator; - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( - indirect_cert, - candidate_index, - )); + if distribute_assignment { + ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( + indirect_cert, + claimed_candidate_indices, + )); + } match approvals_cache.get(&candidate_hash) { Some(ApprovalOutcome::Approved) => { @@ -1061,6 +1071,49 @@ async fn handle_actions( Ok(conclude) } +fn cores_to_candidate_indices( + core_indices: &CoreBitfield, + block_entry: &BlockEntry, +) -> Result { + let mut candidate_indices = Vec::new(); + + // Map from core index to candidate index. + for claimed_core_index in core_indices.iter_ones() { + if let Some(candidate_index) = block_entry + .candidates() + .iter() + .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) + { + candidate_indices.push(candidate_index as CandidateIndex); + } + } + + CandidateBitfield::try_from(candidate_indices) +} + +// Returns the claimed core bitfield from the assignment cert, the candidate hash and a +// `BlockEntry`. Can fail only for VRF Delay assignments for which we cannot find the candidate hash +// in the block entry which indicates a bug or corrupted storage. +fn get_assignment_core_indices( + assignment: &AssignmentCertKindV2, + candidate_hash: &CandidateHash, + block_entry: &BlockEntry, +) -> Option { + match &assignment { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + Some(core_bitfield.clone()), + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => block_entry + .candidates() + .iter() + .find(|(_core_index, h)| candidate_hash == h) + .map(|(core_index, _candidate_hash)| { + CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed") + }), + AssignmentCertKindV2::RelayVRFDelay { core_index } => + Some(CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed")), + } +} + fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, @@ -1125,33 +1178,95 @@ fn distribution_messages_for_activation( match approval_entry.local_statements() { (None, None) | (None, Some(_)) => {}, // second is impossible case. (Some(assignment), None) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + // Should never happen. If we fail here it means the + // assignment is null (no cores claimed). + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + }, + } + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, (Some(assignment), Some(approval_sig)) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + // If we didn't send assignment, we don't send approval. + continue + }, + } - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )) + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVote { + block_hash, + candidate_index: i as _, + validator: assignment.validator_index(), + signature: approval_sig, + }, + )); + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, } }, @@ -1271,14 +1386,14 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_core, res) => { + ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, res) => { let (check_outcome, actions) = check_and_import_assignment( ctx.sender(), state, db, session_info_provider, a, - claimed_core, + claimed_cores, ) .await?; let _ = res.send(check_outcome); @@ -1448,7 +1563,6 @@ async fn handle_approved_ancestor( let mut span = span .child("handle-approved-ancestor") .with_stage(jaeger::Stage::ApprovalChecking); - use bitvec::{order::Lsb0, vec::BitVec}; let mut all_approved_max = None; @@ -1787,8 +1901,8 @@ async fn check_and_import_assignment( state: &State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - assignment: IndirectAssignmentCert, - candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + candidate_indices: CandidateBitfield, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, @@ -1801,9 +1915,12 @@ where .map(|span| span.child("check-and-import-assignment")) .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "check-and-import-assignment")) .with_relay_parent(assignment.block_hash) - .with_uint_tag("candidate-index", candidate_index as u64) .with_stage(jaeger::Stage::ApprovalChecking); + for candidate_index in candidate_indices.iter_ones() { + check_and_import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); + } + let block_entry = match db.load_block_entry(&assignment.block_hash)? { Some(b) => b, None => @@ -1833,39 +1950,64 @@ where )), }; - let (claimed_core_index, assigned_candidate_hash) = - match block_entry.candidate(candidate_index as usize) { - Some((c, h)) => (*c, *h), + let n_cores = session_info.n_cores as usize; + + // Early check the candidate bitfield and core bitfields lengths < `n_cores`. + // Core bitfield length is checked later in `check_assignment_cert`. + if candidate_indices.len() > n_cores { + gum::debug!( + target: LOG_TARGET, + validator = assignment.validator.0, + n_cores, + candidate_bitfield_len = ?candidate_indices.len(), + "Oversized bitfield", + ); + + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_indices.len(), + )), + Vec::new(), + )) + } + + // The Compact VRF modulo assignment cert has multiple core assignments. + let mut backing_groups = Vec::new(); + let mut claimed_core_indices = Vec::new(); + let mut assigned_candidate_hashes = Vec::new(); + + for candidate_index in candidate_indices.iter_ones() { + let (claimed_core_index, assigned_candidate_hash) = + match block_entry.candidate(candidate_index) { + Some((c, h)) => (*c, *h), + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( + candidate_index as _, + )), + Vec::new(), + )), // no candidate at core. + }; + + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, None => return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( - candidate_index, + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + assigned_candidate_hash, )), Vec::new(), )), // no candidate at core. }; - check_and_import_assignment_span - .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); - check_and_import_assignment_span.add_string_tag( - "traceID", - format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), - ); - - let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { - Some(c) => c, - None => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( - candidate_index, - assigned_candidate_hash, - )), - Vec::new(), - )), - }; + check_and_import_assignment_span + .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); + check_and_import_assignment_span.add_string_tag( + "traceID", + format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), + ); - let res = { - // import the assignment. let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, None => @@ -1878,79 +2020,144 @@ where )), }; - let res = state.assignment_criteria.check_assignment_cert( - claimed_core_index, - assignment.validator, - &criteria::Config::from(session_info), - block_entry.relay_vrf_story(), - &assignment.cert, - approval_entry.backing_group(), - ); + backing_groups.push(approval_entry.backing_group()); + claimed_core_indices.push(claimed_core_index); + assigned_candidate_hashes.push(assigned_candidate_hash); + } - let tranche = match res { - Err(crate::criteria::InvalidAssignment(reason)) => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - assignment.validator, - format!("{:?}", reason), - )), - Vec::new(), + // Error on null assignments. + if claimed_core_indices.is_empty() { + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", InvalidAssignmentReason::NullAssignment), + )), + Vec::new(), + )) + } + + // Check the assignment certificate. + let res = state.assignment_criteria.check_assignment_cert( + claimed_core_indices + .clone() + .try_into() + .expect("Checked for null assignment above; qed"), + assignment.validator, + &criteria::Config::from(session_info), + block_entry.relay_vrf_story(), + &assignment.cert, + backing_groups, + ); + + let tranche = match res { + Err(crate::criteria::InvalidAssignment(reason)) => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", reason), )), - Ok(tranche) => { - let current_tranche = - state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); + Vec::new(), + )), + Ok(tranche) => { + let current_tranche = + state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); - let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; + let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - if tranche >= too_far_in_future { - return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) - } + if tranche >= too_far_in_future { + return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) + } - tranche - }, - }; + tranche + }, + }; - check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + let mut actions = Vec::new(); + let res = { + let mut is_duplicate = true; + // Import the assignments for all cores in the cert. + for (assigned_candidate_hash, candidate_index) in + assigned_candidate_hashes.iter().zip(candidate_indices.iter_ones()) + { + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; - let is_duplicate = approval_entry.is_assigned(assignment.validator); - approval_entry.import_assignment(tranche, assignment.validator, tick_now); + let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { + Some(a) => a, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::Internal( + assignment.block_hash, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; + is_duplicate &= approval_entry.is_assigned(assignment.validator); + approval_entry.import_assignment(tranche, assignment.validator, tick_now); + check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + + // We've imported a new assignment, so we need to schedule a wake-up for when that might + // no-show. + if let Some((approval_entry, status)) = state + .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) + .await + { + actions.extend(schedule_wakeup_action( + approval_entry, + block_entry.block_hash(), + block_entry.block_number(), + *assigned_candidate_hash, + status.block_tick, + tick_now, + status.required_tranches, + )); + } + + // We also write the candidate entry as it now contains the new candidate. + db.write_candidate_entry(candidate_entry.into()); + } + // Since we don't account for tranche in distribution message fingerprinting, some + // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be + // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate + // assignments. if is_duplicate { AssignmentCheckResult::AcceptedDuplicate + } else if candidate_indices.count_ones() > 1 { + gum::trace!( + target: LOG_TARGET, + validator = assignment.validator.0, + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + ?tranche, + "Imported assignments for multiple cores.", + ); + + AssignmentCheckResult::Accepted } else { gum::trace!( target: LOG_TARGET, validator = assignment.validator.0, - candidate_hash = ?assigned_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Imported assignment.", + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + "Imported assignment for a single core.", ); AssignmentCheckResult::Accepted } }; - let mut actions = Vec::new(); - - // We've imported a new approval, so we need to schedule a wake-up for when that might no-show. - if let Some((approval_entry, status)) = state - .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) - .await - { - actions.extend(schedule_wakeup_action( - approval_entry, - block_entry.block_hash(), - block_entry.block_number(), - assigned_candidate_hash, - status.block_tick, - tick_now, - status.required_tranches, - )); - } - - // We also write the candidate entry as it now contains the new candidate. - db.write_candidate_entry(candidate_entry.into()); - Ok((res, actions)) } @@ -2324,7 +2531,7 @@ async fn process_wakeup( let candidate_entry = db.load_candidate_entry(&candidate_hash)?; // If either is not present, we have nothing to wakeup. Might have lost a race with finality - let (block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { + let (mut block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { (Some(b), Some(c)) => (b, c), _ => return Ok(Vec::new()), }; @@ -2404,31 +2611,58 @@ async fn process_wakeup( if let Some((cert, val_index, tranche)) = maybe_cert { let indirect_cert = - IndirectAssignmentCert { block_hash: relay_block, validator: val_index, cert }; + IndirectAssignmentCertV2 { block_hash: relay_block, validator: val_index, cert }; + + gum::trace!( + target: LOG_TARGET, + ?candidate_hash, + para_id = ?candidate_receipt.descriptor.para_id, + block_hash = ?relay_block, + "Launching approval work.", + ); - let index_in_candidate = - block_entry.candidates().iter().position(|(_, h)| &candidate_hash == h); + if let Some(claimed_core_indices) = + get_assignment_core_indices(&indirect_cert.cert.kind, &candidate_hash, &block_entry) + { + match cores_to_candidate_indices(&claimed_core_indices, &block_entry) { + Ok(claimed_candidate_indices) => { + // Ensure we distribute multiple core assignments just once. + let distribute_assignment = if claimed_candidate_indices.count_ones() > 1 { + !block_entry.mark_assignment_distributed(claimed_candidate_indices.clone()) + } else { + true + }; + db.write_block_entry(block_entry.clone()); - if let Some(i) = index_in_candidate { - gum::trace!( + actions.push(Action::LaunchApproval { + claimed_candidate_indices, + candidate_hash, + indirect_cert, + assignment_tranche: tranche, + relay_block_hash: relay_block, + session: block_entry.session(), + candidate: candidate_receipt, + backing_group, + distribute_assignment, + }); + }, + Err(err) => { + // Never happens, it should only happen if no cores are claimed, which is a bug. + gum::warn!( + target: LOG_TARGET, + block_hash = ?relay_block, + ?err, + "Failed to create assignment bitfield" + ); + }, + }; + } else { + gum::warn!( target: LOG_TARGET, - ?candidate_hash, - para_id = ?candidate_receipt.descriptor.para_id, block_hash = ?relay_block, - "Launching approval work.", + ?candidate_hash, + "Cannot get assignment claimed core indices", ); - - // sanity: should always be present. - actions.push(Action::LaunchApproval { - candidate_hash, - indirect_cert, - assignment_tranche: tranche, - relay_block_hash: relay_block, - candidate_index: i as _, - session: block_entry.session(), - candidate: candidate_receipt, - backing_group, - }); } } // Although we checked approval earlier in this function, diff --git a/node/core/approval-voting/src/ops.rs b/node/core/approval-voting/src/ops.rs index 6f57b2f80e8a..a6f0ecf9d1f0 100644 --- a/node/core/approval-voting/src/ops.rs +++ b/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v1::{OurAssignment, StoredBlockRange}, + approval_db::v2::{OurAssignment, StoredBlockRange}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/node/core/approval-voting/src/persisted_entries.rs b/node/core/approval-voting/src/persisted_entries.rs index 9b6592220275..155b2f9c4e02 100644 --- a/node/core/approval-voting/src/persisted_entries.rs +++ b/node/core/approval-voting/src/persisted_entries.rs @@ -20,16 +20,21 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche, RelayVRFStory}; +use polkadot_node_primitives::approval::{ + v1::{DelayTranche, RelayVRFStory}, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; -use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice, vec::BitVec}; +use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; use std::collections::BTreeMap; +use crate::approval_db::v2::Bitfield; + use super::{criteria::OurAssignment, time::Tick}; /// Metadata regarding a specific tranche of assignments for a specific candidate. @@ -53,8 +58,8 @@ impl TrancheEntry { } } -impl From for TrancheEntry { - fn from(entry: crate::approval_db::v1::TrancheEntry) -> Self { +impl From for TrancheEntry { + fn from(entry: crate::approval_db::v2::TrancheEntry) -> Self { TrancheEntry { tranche: entry.tranche, assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(), @@ -62,7 +67,7 @@ impl From for TrancheEntry { } } -impl From for crate::approval_db::v1::TrancheEntry { +impl From for crate::approval_db::v2::TrancheEntry { fn from(entry: TrancheEntry) -> Self { Self { tranche: entry.tranche, @@ -80,7 +85,7 @@ pub struct ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, } @@ -92,10 +97,17 @@ impl ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, ) -> Self { - Self { tranches, backing_group, our_assignment, our_approval_sig, assignments, approved } + Self { + tranches, + backing_group, + our_assignment, + our_approval_sig, + assigned_validators, + approved, + } } // Access our assignment for this approval entry. @@ -107,7 +119,7 @@ impl ApprovalEntry { pub fn trigger_our_assignment( &mut self, tick_now: Tick, - ) -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> { + ) -> Option<(AssignmentCertV2, ValidatorIndex, DelayTranche)> { let our = self.our_assignment.as_mut().and_then(|a| { if a.triggered() { return None @@ -131,7 +143,10 @@ impl ApprovalEntry { /// Whether a validator is already assigned. pub fn is_assigned(&self, validator_index: ValidatorIndex) -> bool { - self.assignments.get(validator_index.0 as usize).map(|b| *b).unwrap_or(false) + self.assigned_validators + .get(validator_index.0 as usize) + .map(|b| *b) + .unwrap_or(false) } /// Import an assignment. No-op if already assigned on the same tranche. @@ -158,14 +173,14 @@ impl ApprovalEntry { }; self.tranches[idx].assignments.push((validator_index, tick_now)); - self.assignments.set(validator_index.0 as _, true); + self.assigned_validators.set(validator_index.0 as _, true); } // Produce a bitvec indicating the assignments of all validators up to and // including `tranche`. - pub fn assignments_up_to(&self, tranche: DelayTranche) -> BitVec { + pub fn assignments_up_to(&self, tranche: DelayTranche) -> Bitfield { self.tranches.iter().take_while(|e| e.tranche <= tranche).fold( - bitvec::bitvec![u8, BitOrderLsb0; 0; self.assignments.len()], + bitvec::bitvec![u8, BitOrderLsb0; 0; self.assigned_validators.len()], |mut a, e| { for &(v, _) in &e.assignments { a.set(v.0 as _, true); @@ -193,12 +208,12 @@ impl ApprovalEntry { /// Get the number of validators in this approval entry. pub fn n_validators(&self) -> usize { - self.assignments.len() + self.assigned_validators.len() } /// Get the number of assignments by validators, including the local validator. pub fn n_assignments(&self) -> usize { - self.assignments.count_ones() + self.assigned_validators.count_ones() } /// Get the backing group index of the approval entry. @@ -219,27 +234,27 @@ impl ApprovalEntry { } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v1::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } } -impl From for crate::approval_db::v1::ApprovalEntry { +impl From for crate::approval_db::v2::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } @@ -253,7 +268,7 @@ pub struct CandidateEntry { // Assignments are based on blocks, so we need to track assignments separately // based on the block we are looking at. pub block_assignments: BTreeMap, - pub approvals: BitVec, + pub approvals: Bitfield, } impl CandidateEntry { @@ -290,8 +305,8 @@ impl CandidateEntry { } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v1::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -305,7 +320,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v1::CandidateEntry { +impl From for crate::approval_db::v2::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -336,8 +351,12 @@ pub struct BlockEntry { // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. // The i'th bit is `true` iff the candidate has been approved in the context of this // block. The block can be considered approved if the bitfield has all bits set to `true`. - pub approved_bitfield: BitVec, + pub approved_bitfield: Bitfield, pub children: Vec, + // A list of assignments for which wea already distributed the assignment. + // We use this to ensure we don't distribute multiple core assignments twice as we track + // individual wakeups for each core. + distributed_assignments: Bitfield, } impl BlockEntry { @@ -412,6 +431,39 @@ impl BlockEntry { pub fn parent_hash(&self) -> Hash { self.parent_hash } + + /// Mark distributed assignment for many candidate indices. + /// Returns `true` if an assignment was already distributed for the `candidates`. + pub fn mark_assignment_distributed(&mut self, candidates: CandidateBitfield) -> bool { + let bitfield = candidates.into_inner(); + let total_one_bits = self.distributed_assignments.count_ones(); + + let new_len = std::cmp::max(self.distributed_assignments.len(), bitfield.len()); + self.distributed_assignments.resize(new_len, false); + self.distributed_assignments |= bitfield; + + // If the an operation did not change our current bitfied, we return true. + let distributed = total_one_bits == self.distributed_assignments.count_ones(); + + distributed + } +} + +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } } impl From for BlockEntry { @@ -426,11 +478,12 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: Default::default(), } } } -impl From for crate::approval_db::v1::BlockEntry { +impl From for crate::approval_db::v2::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -442,6 +495,36 @@ impl From for crate::approval_db::v1::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } +} + +/// Migration helpers. +impl From for CandidateEntry { + fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ae.into())) + .collect(), + candidate: value.candidate, + session: value.session, + } + } +} + +impl From for ApprovalEntry { + fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value.our_approval_sig, + assigned_validators: value.assignments, + approved: value.approved, } } } diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs index f58e60c6a487..c2ef109ad4ca 100644 --- a/node/core/approval-voting/src/tests.rs +++ b/node/core/approval-voting/src/tests.rs @@ -15,10 +15,14 @@ // along with Polkadot. If not, see . use super::*; +use crate::backend::V1ReadBackend; use polkadot_node_primitives::{ approval::{ - AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, - RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, + RELAY_VRF_MODULO_CONTEXT, + }, + v2::{AssignmentCertKindV2, AssignmentCertV2}, }, AvailableData, BlockData, PoV, }; @@ -51,7 +55,7 @@ use std::{ }; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -111,7 +115,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v1::Config as DatabaseConfig; + use crate::approval_db::v2::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; @@ -161,7 +165,7 @@ impl Clock for MockClock { // This mock clock allows us to manipulate the time and // be notified when wakeups have been triggered. -#[derive(Default)] +#[derive(Default, Debug)] struct MockClockInner { tick: Tick, wakeups: Vec<(Tick, oneshot::Sender<()>)>, @@ -231,7 +235,7 @@ where fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -244,13 +248,13 @@ where fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, validator_index: ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result { self.1(validator_index) } } @@ -271,6 +275,18 @@ struct TestStoreInner { candidate_entries: HashMap, } +impl V1ReadBackend for TestStoreInner { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStoreInner { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { Ok(self.block_entries.get(block_hash).cloned()) @@ -342,6 +358,18 @@ pub struct TestStore { store: Arc>, } +impl V1ReadBackend for TestStore { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStore { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { let store = self.store.lock(); @@ -391,6 +419,17 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } } +fn garbage_assignment_cert_v2(kind: AssignmentCertKindV2) -> AssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCertV2 { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + fn sign_approval( key: Sr25519Keyring, candidate_hash: CandidateHash, @@ -467,6 +506,12 @@ fn test_harness>( config: HarnessConfig, test: impl FnOnce(TestHarness) -> T, ) { + let _ = env_logger::builder() + .is_test(true) + .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) + .filter(Some(LOG_TARGET), log::LevelFilter::Trace) + .try_init(); + let HarnessConfig { sync_oracle, sync_oracle_handle, clock, backend, assignment_criteria } = config; @@ -616,12 +661,13 @@ async fn check_and_import_assignment( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash, validator, - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), }, - candidate_index, + candidate_index.into(), tx, ), }, @@ -630,6 +676,38 @@ async fn check_and_import_assignment( rx } +async fn check_and_import_assignment_v2( + overseer: &mut VirtualOverseer, + block_hash: Hash, + core_indices: Vec, + validator: ValidatorIndex, +) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + overseer_send( + overseer, + FromOrchestra::Communication { + msg: ApprovalVotingMessage::CheckAndImportAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: core_indices + .clone() + .into_iter() + .map(|c| CoreIndex(c)) + .collect::>() + .try_into() + .unwrap(), + }), + }, + core_indices.try_into().unwrap(), + tx, + ), + }, + ) + .await; + rx +} struct BlockConfig { slot: Slot, candidates: Option>, @@ -742,7 +820,7 @@ fn session_info(keys: &[Sr25519Keyring]) -> SessionInfo { vec![ValidatorIndex(0)], vec![ValidatorIndex(1)], ]), - n_cores: keys.len() as _, + n_cores: 10, needed_approvals: 2, zeroth_delay_tranche_width: 5, relay_vrf_modulo_samples: 3, @@ -1059,14 +1137,15 @@ fn blank_subsystem_act_on_bad_block() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: bad_block_hash, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1322,9 +1401,22 @@ fn subsystem_accepts_duplicate_assignment() { } ); - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; let validator = ValidatorIndex(0); + let candidate_index = 0; + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; // Add block hash 00. ChainBuilder::new() @@ -1332,21 +1424,30 @@ fn subsystem_accepts_duplicate_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: Some(vec![ + (candidate_receipt1, CoreIndex(0), GroupIndex(1)), + (candidate_receipt2, CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, ) .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( + // Initial assignment. + let rx = check_and_import_assignment_v2( &mut virtual_overseer, block_hash, - candidate_index, + vec![candidate_index1, candidate_index2], validator, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); + // Test with single assigned core. let rx = check_and_import_assignment( &mut virtual_overseer, block_hash, @@ -1357,6 +1458,18 @@ fn subsystem_accepts_duplicate_assignment() { assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + // Test with multiple assigned cores. This cannot happen in practice, as tranche0 + // assignments are sent first, but we should still ensure correct behavior. + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![candidate_index1, candidate_index2], + validator, + ) + .await; + + assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + virtual_overseer }); } @@ -1407,6 +1520,63 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { }); } +#[test] +fn subsystem_rejects_oversized_bitfields() { + test_harness(HarnessConfig::default(), |test_harness| async move { + let TestHarness { mut virtual_overseer, sync_oracle_handle: _sync_oracle_handle, .. } = + test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let candidate_index = 10; + let validator = ValidatorIndex(0); + + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + ) + .build(&mut virtual_overseer) + .await; + + let rx = check_and_import_assignment( + &mut virtual_overseer, + block_hash, + candidate_index, + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_index as usize + 1 + ))), + ); + + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![1, 2, 10, 50], + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield(51))), + ); + virtual_overseer + }); +} + #[test] fn subsystem_accepts_and_imports_approval_after_assignment() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1727,14 +1897,15 @@ fn linear_import_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1797,14 +1968,15 @@ fn forkful_import_at_same_height_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -2248,8 +2420,24 @@ fn subsystem_validate_approvals_cache() { let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), tranche: 0, validator_index: ValidatorIndex(0), triggered: false, @@ -2346,6 +2534,137 @@ fn subsystem_validate_approvals_cache() { }); } +#[test] +fn subsystem_doesnt_distribute_duplicate_compact_assignments() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let cert = garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(), + }); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert, + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { + mut virtual_overseer, + sync_oracle_handle: _sync_oracle_handle, + clock, + .. + } = test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; + + // Add block hash 00. + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot: Slot::from(0), + candidates: Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(1)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, + ) + .build(&mut virtual_overseer) + .await; + + // Activate the wakeup present above, and sleep to allow process_wakeups to execute.. + assert_eq!(Some(2), clock.inner.lock().next_wakeup()); + gum::trace!("clock \n{:?}\n", clock.inner.lock()); + + clock.inner.lock().wakeup_all(100); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(100)).await; + + // Assignment is distributed only once from `approval-voting` + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(c_indices, vec![candidate_index1, candidate_index2].try_into().unwrap()); + } + ); + + // Candidate 1 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Candidate 2 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Check if assignment was triggered for candidate 1. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt1.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + // Check if assignment was triggered for candidate 2. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt2.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + virtual_overseer + }); +} + /// Ensure that when two assignments are imported, only one triggers the Approval Checking work async fn handle_double_assignment_import( virtual_overseer: &mut VirtualOverseer, @@ -2355,9 +2674,9 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( _, - c_index, + c_indices, )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_indices); } ); @@ -2370,7 +2689,7 @@ async fn handle_double_assignment_import( _, c_index )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_index); } ); @@ -2391,7 +2710,6 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); - // Assert that there are no more messages being sent by the subsystem assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); } @@ -2466,8 +2784,9 @@ where let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), tranche: our_assigned_tranche, validator_index: ValidatorIndex(0), triggered: false, @@ -2601,14 +2920,12 @@ async fn step_until_done(clock: &MockClock) { futures_timer::Delay::new(Duration::from_millis(200)).await; let mut clock = clock.inner.lock(); if let Some(tick) = clock.next_wakeup() { - println!("TICK: {:?}", tick); relevant_ticks.push(tick); clock.set_tick(tick); } else { break } } - println!("relevant_ticks: {:?}", relevant_ticks); } #[test] diff --git a/node/core/approval-voting/src/time.rs b/node/core/approval-voting/src/time.rs index 34132dc22b23..a45866402c82 100644 --- a/node/core/approval-voting/src/time.rs +++ b/node/core/approval-voting/src/time.rs @@ -17,7 +17,7 @@ //! Time utilities for approval voting. use futures::prelude::*; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ pin::Pin, diff --git a/node/core/dispute-coordinator/src/import.rs b/node/core/dispute-coordinator/src/import.rs index 0da3723ebf22..6222aab1cb10 100644 --- a/node/core/dispute-coordinator/src/import.rs +++ b/node/core/dispute-coordinator/src/import.rs @@ -109,7 +109,7 @@ pub enum OwnVoteState { } impl OwnVoteState { - fn new(votes: &CandidateVotes, env: &CandidateEnvironment<'_>) -> Self { + fn new(votes: &CandidateVotes, env: &CandidateEnvironment) -> Self { let controlled_indices = env.controlled_indices(); if controlled_indices.is_empty() { return Self::CannotVote diff --git a/node/network/approval-distribution/Cargo.toml b/node/network/approval-distribution/Cargo.toml index bd683b320285..113b9429c911 100644 --- a/node/network/approval-distribution/Cargo.toml +++ b/node/network/approval-distribution/Cargo.toml @@ -14,10 +14,12 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-jaeger = { path = "../../jaeger" } rand = "0.8" +itertools = "0.10.5" futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [dev-dependencies] sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index ac525ea6faf3..746a4b4dab5c 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -20,17 +20,23 @@ #![warn(missing_docs)] +use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; +use itertools::Itertools; +use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, - peer_set::{ValidationVersion, MAX_NOTIFICATION_SIZE}, + peer_set::MAX_NOTIFICATION_SIZE, v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, UnifiedReputationChange as Rep, - Versioned, VersionedValidationProtocol, View, + Versioned, View, }; use polkadot_node_primitives::approval::{ - AssignmentCert, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{ + AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + }, + v2::{AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2}, }; use polkadot_node_subsystem::{ messages::{ @@ -49,8 +55,6 @@ use std::{ time::Duration, }; -use self::metrics::Metrics; - mod metrics; #[cfg(test)] @@ -64,11 +68,15 @@ const COST_DUPLICATE_MESSAGE: Rep = Rep::CostMinorRepeated("Peer sent identical const COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE: Rep = Rep::CostMinor("The vote was valid but too far in the future"); const COST_INVALID_MESSAGE: Rep = Rep::CostMajor("The vote was bad"); +const COST_OVERSIZED_BITFIELD: Rep = Rep::CostMajor("Oversized certificate or candidate bitfield"); const BENEFIT_VALID_MESSAGE: Rep = Rep::BenefitMinor("Peer sent a valid message"); const BENEFIT_VALID_MESSAGE_FIRST: Rep = Rep::BenefitMinorFirst("Valid message with new information"); +// Maximum valid size for the `CandidateBitfield` in the assignment messages. +const MAX_BITFIELD_SIZE: usize = 500; + /// The Approval Distribution subsystem. pub struct ApprovalDistribution { metrics: Metrics, @@ -97,7 +105,144 @@ impl RecentlyOutdated { } } -// In case the original gtid topology mechanisms don't work on their own, we need to trade bandwidth +// Contains topology routing information for assignments and approvals. +struct ApprovalRouting { + required_routing: RequiredRouting, + local: bool, + random_routing: RandomRouting, +} + +// This struct is responsible for tracking the full state of an assignment and grid routing +// information. +struct ApprovalEntry { + // The assignment certificate. + assignment: IndirectAssignmentCertV2, + // The candidates claimed by the certificate. A mapping between bit index and candidate index. + candidates: CandidateBitfield, + // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. + approvals: HashMap, + // The validator index of the assignment signer. + validator_index: ValidatorIndex, + // Information required for gossiping to other peers using the grid topology. + routing_info: ApprovalRouting, +} + +#[derive(Debug)] +enum ApprovalEntryError { + InvalidValidatorIndex, + CandidateIndexOutOfBounds, + InvalidCandidateIndex, + DuplicateApproval, +} + +impl ApprovalEntry { + pub fn new( + assignment: IndirectAssignmentCertV2, + candidates: CandidateBitfield, + routing_info: ApprovalRouting, + ) -> ApprovalEntry { + Self { + validator_index: assignment.validator, + assignment, + approvals: HashMap::with_capacity(candidates.len()), + candidates, + routing_info, + } + } + + // Create a `MessageSubject` to reference the assignment. + pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageKind::Assignment, + ) + } + + // Create a `MessageSubject` to reference the approval. + pub fn create_approval_knowledge( + &self, + block_hash: Hash, + candidate_index: CandidateIndex, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, candidate_index.into(), self.validator_index), + MessageKind::Approval, + ) + } + + // Updates routing information and returns the previous information if any. + pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { + &mut self.routing_info + } + + // Get the routing information. + pub fn routing_info(&self) -> &ApprovalRouting { + &self.routing_info + } + + // Update routing information. + pub fn update_required_routing(&mut self, required_routing: RequiredRouting) { + self.routing_info.required_routing = required_routing; + } + + // Records a new approval. Returns false if the claimed candidate is not found or we already + // have received the approval. + pub fn note_approval( + &mut self, + approval: IndirectSignedApprovalVote, + ) -> Result<(), ApprovalEntryError> { + // First do some sanity checks: + // - check validator index matches + // - check claimed candidate + // - check for duplicate approval + if self.validator_index != approval.validator { + return Err(ApprovalEntryError::InvalidValidatorIndex) + } + + if self.candidates.len() <= approval.candidate_index as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + if !self.candidates.bit_at(approval.candidate_index.as_bit_index()) { + return Err(ApprovalEntryError::InvalidCandidateIndex) + } + + if self.approvals.contains_key(&approval.candidate_index) { + return Err(ApprovalEntryError::DuplicateApproval) + } + + self.approvals.insert(approval.candidate_index, approval); + Ok(()) + } + + // Get the assignment certiticate and claimed candidates. + pub fn assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { + (self.assignment.clone(), self.candidates.clone()) + } + + // Get all approvals for all candidates claimed by the assignment. + pub fn approvals(&self) -> Vec { + self.approvals.values().cloned().collect::>() + } + + // Get the approval for a specific candidate index. + pub fn approval(&self, candidate_index: CandidateIndex) -> Option { + self.approvals.get(&candidate_index).cloned() + } + + // Get validator index. + pub fn validator_index(&self) -> ValidatorIndex { + self.validator_index + } +} + +// We keep track of each peer view and protocol version using this struct. +struct PeerEntry { + pub view: View, + pub version: ProtocolVersion, +} + +// In case the original grid topology mechanisms don't work on their own, we need to trade bandwidth // for protocol liveliness by introducing aggression. // // Aggression has 3 levels: @@ -117,7 +262,6 @@ impl RecentlyOutdated { // not to all blocks older than the threshold. Most likely, a few assignments struggle to // be propagated in a single block and this holds up all of its descendants blocks. // Accordingly, we only step on the gas for the block which is most obviously holding up finality. - /// Aggression configuration representation #[derive(Clone)] struct AggressionConfig { @@ -160,15 +304,6 @@ enum Resend { No, } -/// Data stored on a per-peer basis. -#[derive(Debug)] -struct PeerData { - /// The peer's view. - view: View, - /// The peer's protocol version. - version: ValidationVersion, -} - /// The [`State`] struct is responsible for tracking the overall state of the subsystem. /// /// It tracks metadata about our view of the unfinalized chain, @@ -189,7 +324,7 @@ struct State { pending_known: HashMap>, /// Peer data is partially stored here, and partially inline within the [`BlockEntry`]s - peer_data: HashMap, + peer_views: HashMap, /// Keeps a topology for various different sessions. topologies: SessionGridTopologies, @@ -197,12 +332,12 @@ struct State { /// Tracks recently finalized blocks. recent_outdated_blocks: RecentlyOutdated, - /// Config for aggression. - aggression_config: AggressionConfig, - - /// `HashMap` from active leaves to spans + /// HashMap from active leaves to spans spans: HashMap, + /// Aggression configuration. + aggression_config: AggressionConfig, + /// Current approval checking finality lag. approval_checking_lag: BlockNumber, @@ -216,8 +351,11 @@ enum MessageKind { Approval, } +// Utility structure to identify assignments and approvals for specific candidates. +// Assignments can span multiple candidates, while approvals refer to only one candidate. +// #[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct MessageSubject(Hash, CandidateIndex, ValidatorIndex); +struct MessageSubject(Hash, pub CandidateBitfield, ValidatorIndex); #[derive(Debug, Clone, Default)] struct Knowledge { @@ -238,9 +376,11 @@ impl Knowledge { } fn insert(&mut self, message: MessageSubject, kind: MessageKind) -> bool { - match self.known_messages.entry(message) { + let mut success = match self.known_messages.entry(message.clone()) { hash_map::Entry::Vacant(vacant) => { vacant.insert(kind); + // If there are multiple candidates assigned in the message, create + // separate entries for each one. true }, hash_map::Entry::Occupied(mut occupied) => match (*occupied.get(), kind) { @@ -252,7 +392,25 @@ impl Knowledge { true }, }, + }; + + // In case of succesful insertion of multiple candidate assignments create additional + // entries for each assigned candidate. This fakes knowledge of individual assignments, but + // we need to share the same `MessageSubject` with the followup approval candidate index. + if kind == MessageKind::Assignment && success && message.1.count_ones() > 1 { + for candidate_index in message.1.iter_ones() { + success = success && + self.insert( + MessageSubject( + message.0, + vec![candidate_index as u32].try_into().expect("Non-empty vec; qed"), + message.2, + ), + kind, + ); + } } + success } } @@ -286,47 +444,97 @@ struct BlockEntry { candidates: Vec, /// The session index of this block. session: SessionIndex, + /// Approval entries for whole block. These also contain all approvals in the case of multiple + /// candidates being claimed by assignments. + approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, } -#[derive(Debug)] -enum ApprovalState { - Assigned(AssignmentCert), - Approved(AssignmentCert, ValidatorSignature), -} +impl BlockEntry { + // Returns the peer which currently know this block. + pub fn known_by(&self) -> Vec { + self.known_by.keys().cloned().collect::>() + } -impl ApprovalState { - fn assignment_cert(&self) -> &AssignmentCert { - match *self { - ApprovalState::Assigned(ref cert) => cert, - ApprovalState::Approved(ref cert, _) => cert, + pub fn insert_approval_entry(&mut self, entry: ApprovalEntry) -> &mut ApprovalEntry { + // First map one entry per candidate to the same key we will use in `approval_entries`. + // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) + // entry in `candidate_entry.messages`. + for claimed_candidate_index in entry.candidates.iter_ones() { + match self.candidates.get_mut(claimed_candidate_index) { + Some(candidate_entry) => { + candidate_entry + .messages + .entry(entry.validator_index()) + .or_insert(entry.candidates.clone()); + }, + None => { + // This should never happen, but if it happens, it means the subsystem is + // broken. + gum::warn!( + target: LOG_TARGET, + hash = ?entry.assignment.block_hash, + ?claimed_candidate_index, + "Missing candidate entry on `import_and_circulate_assignment`", + ); + }, + }; } + + self.approval_entries + .entry((entry.validator_index, entry.candidates.clone())) + .or_insert(entry) } - fn approval_signature(&self) -> Option { - match *self { - ApprovalState::Assigned(_) => None, - ApprovalState::Approved(_, ref sig) => Some(sig.clone()), - } + // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator + // `validator_index`. + pub fn approval_entry( + &mut self, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + ) -> Option<&mut ApprovalEntry> { + self.candidates + .get(candidate_index as usize) + .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) + .map_or(None, |candidate_indices| { + self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + }) } -} -// routing state bundled with messages for the candidate. Corresponding assignments -// and approvals are stored together and should be routed in the same way, with -// assignments preceding approvals in all cases. -#[derive(Debug)] -struct MessageState { - required_routing: RequiredRouting, - local: bool, - random_routing: RandomRouting, - approval_state: ApprovalState, + // Get all approval entries for a given candidate. + pub fn approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { + // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, + let approval_entry_keys = self + .candidates + .get(candidate_index as usize) + .map(|candidate_entry| &candidate_entry.messages); + + if let Some(approval_entry_keys) = approval_entry_keys { + // Ensure no duplicates. + let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); + + let mut entries = Vec::new(); + for (validator_index, candidate_indices) in approval_entry_keys { + if let Some(entry) = + self.approval_entries.get(&(*validator_index, candidate_indices.clone())) + { + entries.push(entry); + } + } + entries + } else { + vec![] + } + } } -/// Information about candidates in the context of a particular block they are included in. -/// In other words, multiple `CandidateEntry`s may exist for the same candidate, -/// if it is included by multiple blocks - this is likely the case when there are forks. +// Information about candidates in the context of a particular block they are included in. +// In other words, multiple `CandidateEntry`s may exist for the same candidate, +// if it is included by multiple blocks - this is likely the case when there are forks. #[derive(Debug, Default)] struct CandidateEntry { - messages: HashMap, + // The value represents part of the lookup key in `approval_entries` to fetch the assignment + // and existing votes. + messages: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -345,7 +553,7 @@ impl MessageSource { } enum PendingMessage { - Assignment(IndirectAssignmentCert, CandidateIndex), + Assignment(IndirectAssignmentCertV2, CandidateBitfield), Approval(IndirectSignedApprovalVote), } @@ -362,27 +570,13 @@ impl State { NetworkBridgeEvent::PeerConnected(peer_id, role, version, _) => { // insert a blank view if none already present gum::trace!(target: LOG_TARGET, ?peer_id, ?role, "Peer connected"); - let version = match ValidationVersion::try_from(version).ok() { - Some(v) => v, - None => { - // sanity: network bridge is supposed to detect this already. - gum::error!( - target: LOG_TARGET, - ?peer_id, - ?version, - "Unsupported protocol version" - ); - return - }, - }; - - self.peer_data + self.peer_views .entry(peer_id) - .or_insert_with(|| PeerData { version, view: Default::default() }); + .or_insert(PeerEntry { view: Default::default(), version }); }, NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); - self.peer_data.remove(&peer_id); + self.peer_views.remove(&peer_id); self.blocks.iter_mut().for_each(|(_hash, entry)| { entry.known_by.remove(&peer_id); }) @@ -419,8 +613,8 @@ impl State { live }); }, - NetworkBridgeEvent::PeerMessage(peer_id, msg) => { - self.process_incoming_peer_message(ctx, metrics, peer_id, msg, rng).await; + NetworkBridgeEvent::PeerMessage(peer_id, message) => { + self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. @@ -459,6 +653,7 @@ impl State { knowledge: Knowledge::default(), candidates, session: meta.session, + approval_entries: HashMap::new(), }); self.topologies.inc_session_refs(meta.session); @@ -481,18 +676,17 @@ impl State { { let sender = ctx.sender(); - for (peer_id, data) in self.peer_data.iter() { - let intersection = data.view.iter().filter(|h| new_hashes.contains(h)); - let view_intersection = - View::new(intersection.cloned(), data.view.finalized_number); + for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { + let intersection = view.iter().filter(|h| new_hashes.contains(h)); + let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( sender, metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), *peer_id, - data.version, + *version, view_intersection, rng, ) @@ -530,13 +724,13 @@ impl State { for (peer_id, message) in to_import { match message { - PendingMessage::Assignment(assignment, claimed_index) => { + PendingMessage::Assignment(assignment, claimed_indices) => { self.import_and_circulate_assignment( ctx, metrics, MessageSource::Peer(peer_id), assignment, - claimed_index, + claimed_indices, rng, ) .await; @@ -575,33 +769,78 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| block_entry.session == session, |required_routing, local, validator_index| { - if *required_routing == RequiredRouting::PendingTopology { - *required_routing = topology + if required_routing == &RequiredRouting::PendingTopology { + topology .local_grid_neighbors() - .required_routing_by_index(*validator_index, local); + .required_routing_by_index(*validator_index, local) + } else { + *required_routing } }, + &self.peer_views, ) .await; } + async fn process_incoming_assignments( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + rng: &mut R, + ) where + R: CryptoRng + Rng, + { + for (assignment, claimed_indices) in assignments { + if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { + let block_hash = &assignment.block_hash; + let validator_index = assignment.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?claimed_indices, + ?validator_index, + "Pending assignment", + ); + + pending.push((peer_id, PendingMessage::Assignment(assignment, claimed_indices))); + + continue + } + + self.import_and_circulate_assignment( + ctx, + metrics, + MessageSource::Peer(peer_id), + assignment, + claimed_indices, + rng, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: Versioned< + protocol_v1::ApprovalDistributionMessage, + protocol_vstaging::ApprovalDistributionMessage, + >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( assignments, )) => { @@ -611,42 +850,42 @@ impl State { num = assignments.len(), "Processing assignments from a peer", ); - for (assignment, claimed_index) in assignments.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { - let message_subject = MessageSubject( - assignment.block_hash, - claimed_index, - assignment.validator, - ); - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?message_subject, - "Pending assignment", - ); + let sanitized_assignments = + self.sanitize_v2_assignments(peer_id, ctx.sender(), assignments).await; - pending - .push((peer_id, PendingMessage::Assignment(assignment, claimed_index))); + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; + }, + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) => { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = assignments.len(), + "Processing assignments from a peer", + ); - continue - } + let sanitized_assignments = + self.sanitize_v1_assignments(peer_id, ctx.sender(), assignments).await; - self.import_and_circulate_assignment( - ctx, - metrics, - MessageSource::Peer(peer_id), - assignment, - claimed_index, - rng, - ) - .await; - } + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; }, - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( approvals, - )) => { + )) | + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, @@ -655,17 +894,17 @@ impl State { ); for approval_vote in approvals.into_iter() { if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let message_subject = MessageSubject( - approval_vote.block_hash, - approval_vote.candidate_index, - approval_vote.validator, - ); + let block_hash = approval_vote.block_hash; + let candidate_index = approval_vote.candidate_index; + let validator_index = approval_vote.validator; gum::trace!( target: LOG_TARGET, %peer_id, - ?message_subject, - "Pending approval", + ?block_hash, + ?candidate_index, + ?validator_index, + "Pending assignment", ); pending.push((peer_id, PendingMessage::Approval(approval_vote))); @@ -699,14 +938,23 @@ impl State { { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; - let (peer_protocol_version, old_finalized_number) = match self - .peer_data - .get_mut(&peer_id) - .map(|d| (d.version, std::mem::replace(&mut d.view, view.clone()))) - { - Some((v, view)) => (v, view.finalized_number), - None => return, // unknown peer - }; + + let (old_view, protocol_version) = + if let Some(peer_entry) = self.peer_views.get_mut(&peer_id) { + (Some(std::mem::replace(&mut peer_entry.view, view.clone())), peer_entry.version) + } else { + // This shouldn't happen, but if it does we assume protocol version 1. + gum::warn!( + target: LOG_TARGET, + ?peer_id, + ?view, + "Peer view change for missing `peer_entry`" + ); + + (None, ValidationVersion::V1.into()) + }; + + let old_finalized_number = old_view.map(|v| v.finalized_number).unwrap_or(0); // we want to prune every block known_by peer up to (including) view.finalized_number let blocks = &mut self.blocks; @@ -731,9 +979,9 @@ impl State { metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), peer_id, - peer_protocol_version, + protocol_version, view, rng, ) @@ -774,8 +1022,8 @@ impl State { ctx: &mut Context, metrics: &Metrics, source: MessageSource, - assignment: IndirectAssignmentCert, - claimed_candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + claimed_candidate_indices: CandidateBitfield, rng: &mut R, ) where R: CryptoRng + Rng, @@ -823,9 +1071,11 @@ impl State { }, }; - // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, claimed_candidate_index, validator_index); - let message_kind = MessageKind::Assignment; + // Compute metadata on the assignment. + let (message_subject, message_kind) = ( + MessageSubject(block_hash, claimed_candidate_indices.clone(), validator_index), + MessageKind::Assignment, + ); if let Some(peer_id) = source.peer_id() { // check if our knowledge of the peer already contains this assignment @@ -841,6 +1091,7 @@ impl State { ?message_subject, "Duplicate assignment", ); + modify_reputation( &mut self.reputation, ctx.sender(), @@ -848,6 +1099,15 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + } else { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + hash = ?block_hash, + ?validator_index, + ?message_subject, + "We sent the message to the peer while peer was sending it to us. Known race condition.", + ); } return } @@ -889,7 +1149,7 @@ impl State { ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( assignment.clone(), - claimed_candidate_index, + claimed_candidate_indices.clone(), tx, )) .await; @@ -920,7 +1180,7 @@ impl State { BENEFIT_VALID_MESSAGE_FIRST, ) .await; - entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); + entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); } @@ -994,7 +1254,7 @@ impl State { // Invariant: to our knowledge, none of the peers except for the `source` know about the // assignment. - metrics.on_assignment_imported(); + metrics.on_assignment_imported(&assignment.cert.kind); let topology = self.topologies.get_topology(entry.session); let local = source == MessageSource::Local; @@ -1003,28 +1263,14 @@ impl State { t.local_grid_neighbors().required_routing_by_index(validator_index, local) }); - let message_state = match entry.candidates.get_mut(claimed_candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Assigned - // unless the approval state is set already - candidate_entry.messages.entry(validator_index).or_insert_with(|| MessageState { - required_routing, - local, - random_routing: Default::default(), - approval_state: ApprovalState::Assigned(assignment.cert.clone()), - }) - }, - None => { - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?claimed_candidate_index, - "Expected a candidate entry on import_and_circulate_assignment", - ); + // All the peers that know the relay chain block. + let peers_to_filter = entry.known_by(); - return - }, - }; + let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( + assignment.clone(), + claimed_candidate_indices.clone(), + ApprovalRouting { required_routing, local, random_routing: Default::default() }, + )); // Dispatch the message to all peers in the routing set which // know the block. @@ -1032,83 +1278,65 @@ impl State { // If the topology isn't known yet (race with networking subsystems) // then messages will be sent when we get it. - let assignments = vec![(assignment, claimed_candidate_index)]; - let n_peers_total = self.peer_data.len(); + let assignments = vec![(assignment, claimed_candidate_indices.clone())]; + let n_peers_total = self.peer_views.len(); let source_peer = source.peer_id(); - let mut peer_filter = move |peer| { - if Some(peer) == source_peer.as_ref() { - return false + // Peers that we will send the assignment to. + let mut peers = Vec::new(); + + // Filter destination peers + for peer in peers_to_filter.into_iter() { + if Some(peer) == source_peer { + continue } if let Some(true) = topology .as_ref() - .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, peer)) + .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, &peer)) { - return true + peers.push(peer); + continue } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. - let route_random = message_state.random_routing.sample(n_peers_total, rng); + let route_random = + approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - message_state.random_routing.inc_sent(); - } - - route_random - }; - - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .keys() - .filter_map(|p| peer_data.get_key_value(p)) - .filter(|(p, _)| peer_filter(p)) - .map(|(p, peer_data)| (*p, peer_data.version)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + approval_entry.routing_info_mut().random_routing.inc_sent(); + peers.push(peer); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?claimed_candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an assignment to peers", - ); + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { + peer_knowledge.sent.insert(message_subject.clone(), message_kind); } + } - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); + if !peers.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?claimed_candidate_indices, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an assignment to peers", + ); - (v1_peers, vstaging_peers) - }; + let peers = peers + .iter() + .filter_map(|peer_id| { + self.peer_views.get(peer_id).map(|peer_entry| (*peer_id, peer_entry.version)) + }) + .collect::>(); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_assignments_packet(ValidationVersion::V1, assignments.clone()), - )) - .await; - } - - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_assignments_packet(ValidationVersion::VStaging, assignments.clone()), - )) - .await; + send_assignments_batched(ctx.sender(), assignments, &peers).await; } } @@ -1143,6 +1371,14 @@ impl State { _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?block_hash, + ?validator_index, + ?candidate_index, + "Approval from a peer is out of view", + ); modify_reputation( &mut self.reputation, ctx.sender(), @@ -1157,7 +1393,7 @@ impl State { }; // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index, validator_index); + let message_subject = MessageSubject(block_hash, candidate_index.into(), validator_index); let message_kind = MessageKind::Approval; if let Some(peer_id) = source.peer_id() { @@ -1307,71 +1543,48 @@ impl State { } } - // Invariant: to our knowledge, none of the peers except for the `source` know about the - // approval. - metrics.on_approval_imported(); - - let required_routing = match entry.candidates.get_mut(candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Approved - // it should be in assigned state already - match candidate_entry.messages.remove(&validator_index) { - Some(MessageState { - approval_state: ApprovalState::Assigned(cert), - required_routing, - local, - random_routing, - }) => { - candidate_entry.messages.insert( - validator_index, - MessageState { - approval_state: ApprovalState::Approved( - cert, - vote.signature.clone(), - ), - required_routing, - local, - random_routing, - }, - ); - - required_routing - }, - Some(_) => { - unreachable!( - "we only insert it after the metadata, checked the metadata above; qed" - ); - }, - None => { - // this would indicate a bug in approval-voting - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Importing an approval we don't have an assignment for", - ); + let required_routing = match entry.approval_entry(candidate_index, validator_index) { + Some(approval_entry) => { + // Invariant: to our knowledge, none of the peers except for the `source` know about + // the approval. + metrics.on_approval_imported(); + + if let Err(err) = approval_entry.note_approval(vote.clone()) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval + gum::warn!( + target: LOG_TARGET, + hash = ?block_hash, + ?candidate_index, + ?validator_index, + ?err, + "Possible bug: Vote import failed", + ); - return - }, + return } + + approval_entry.routing_info().required_routing }, None => { + let peer_id = source.peer_id(); + // This indicates a bug in approval-distribution, since we check the knowledge at + // the begining of the function. gum::warn!( target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Expected a candidate entry on import_and_circulate_approval", + ?peer_id, + ?message_subject, + "Unknown approval assignment", ); - + // No rep change as this is caused by an issue return }, }; // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. - let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); @@ -1395,57 +1608,56 @@ impl State { in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) }; - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .iter() - .filter_map(|(p, k)| peer_data.get(&p).map(|pd| (p, k, pd.version))) - .filter(|(p, k, _)| peer_filter(p, k)) - .map(|(p, _, v)| (*p, v)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + let peers = entry + .known_by + .iter() + .filter(|(p, k)| peer_filter(p, k)) + .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) + .collect::>(); + + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(entry) = entry.known_by.get_mut(&peer.0) { + entry.sent.insert(message_subject.clone(), message_kind); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an approval to peers", - ); - } - - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); - - (v1_peers, vstaging_peers) - }; + if !peers.is_empty() { + let approvals = vec![vote]; + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?candidate_index, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an approval to peers", + ); - let approvals = vec![vote]; + let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_approvals_packet(ValidationVersion::V1, approvals.clone()), - )) - .await; - } + if !v1_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals.clone()), + )), + )) + .await; + } - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_approvals_packet(ValidationVersion::VStaging, approvals), - )) - .await; + if !v2_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers, + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), + ), + ), + )) + .await; + } } } @@ -1476,25 +1688,12 @@ impl State { Some(e) => e, }; - let candidate_entry = match block_entry.candidates.get(index as usize) { - None => { - gum::debug!( - target: LOG_TARGET, - ?hash, - ?index, - "`get_approval_signatures`: could not find candidate entry for given hash and index!" - ); - continue - }, - Some(e) => e, - }; - let sigs = - candidate_entry.messages.iter().filter_map(|(validator_index, message_state)| { - match &message_state.approval_state { - ApprovalState::Approved(_, sig) => Some((*validator_index, sig.clone())), - ApprovalState::Assigned(_) => None, - } - }); + let sigs = block_entry + .approval_entries(index) + .into_iter() + .filter_map(|approval_entry| approval_entry.approval(index)) + .map(|approval| (approval.validator, approval.signature)) + .collect::>(); all_sigs.extend(sigs); } all_sigs @@ -1507,7 +1706,7 @@ impl State { topologies: &SessionGridTopologies, total_peers: usize, peer_id: PeerId, - peer_protocol_version: ValidationVersion, + protocol_version: ProtocolVersion, view: View, rng: &mut (impl CryptoRng + Rng), ) { @@ -1520,6 +1719,8 @@ impl State { let view_finalized_number = view.finalized_number; for head in view.into_iter() { let mut block = head; + + // Walk the chain back to last finalized block of the peer view. loop { let entry = match entries.get_mut(&block) { Some(entry) if entry.number > view_finalized_number => entry, @@ -1534,19 +1735,16 @@ impl State { } let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - // Iterate all messages in all candidates. - for (candidate_index, validator, message_state) in - entry.candidates.iter_mut().enumerate().flat_map(|(c_i, c)| { - c.messages.iter_mut().map(move |(k, v)| (c_i as _, k, v)) - }) { + // We want to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { // Propagate the message to all peers in the required routing set OR // randomly sample peers. { - let random_routing = &mut message_state.random_routing; - let required_routing = message_state.required_routing; + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = &mut approval_entry.routing_info_mut().random_routing; let rng = &mut *rng; let mut peer_filter = move |peer_id| { let in_topology = topology.as_ref().map_or(false, |t| { @@ -1567,39 +1765,24 @@ impl State { } } - let message_subject = MessageSubject(block, candidate_index, *validator); - - let assignment_message = ( - IndirectAssignmentCert { - block_hash: block, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: block, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); assignments_to_send.push(assignment_message); } - if let Some(approval_message) = approval_message { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(block, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); approvals_to_send.push(approval_message); } } @@ -1613,23 +1796,30 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = assignments_to_send.len(), "Sending assignments to unified peer", ); - send_assignments_batched(sender, assignments_to_send, peer_id, peer_protocol_version) - .await; + send_assignments_batched( + sender, + assignments_to_send, + &vec![(peer_id, protocol_version)], + ) + .await; } if !approvals_to_send.is_empty() { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = approvals_to_send.len(), "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, peer_id, peer_protocol_version).await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -1665,7 +1855,6 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1687,13 +1876,13 @@ impl State { false } }, - |_, _, _| {}, + |required_routing, _, _| *required_routing, + &self.peer_views, ) .await; adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1712,30 +1901,110 @@ impl State { lag = ?self.approval_checking_lag, "Encountered old block pending gossip topology", ); - return + return *required_routing } + let mut new_required_routing = *required_routing; + if config.l1_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. - if local && *required_routing != RequiredRouting::All { + if local && new_required_routing != RequiredRouting::All { metrics.on_aggression_l1(); - *required_routing = RequiredRouting::All; + new_required_routing = RequiredRouting::All; } } if config.l2_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. Everyone else sends to XY. - if !local && *required_routing != RequiredRouting::GridXY { + if !local && new_required_routing != RequiredRouting::GridXY { metrics.on_aggression_l2(); - *required_routing = RequiredRouting::GridXY; + new_required_routing = RequiredRouting::GridXY; } } + new_required_routing }, + &self.peer_views, ) .await; } + + // Filter out invalid candidate index and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v1_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_index) in assignments.into_iter() { + let cert_bitfield_bits = match cert.cert.kind { + AssignmentCertKind::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKind::RelayVRFModulo { .. } => candidate_index as usize + 1, + }; + + let candidate_bitfield_bits = candidate_index as usize + 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE || candidate_bitfield_bits > MAX_BITFIELD_SIZE + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert.into(), candidate_index.into())) + } + } + + sanitized_assignments + } + + // Filter out oversized candidate and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v2_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_bitfield) in assignments.into_iter() { + let cert_bitfield_bits = match &cert.cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKindV2::RelayVRFModulo { .. } => candidate_bitfield.len(), + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.len(), + }; + + let candidate_bitfield_bits = candidate_bitfield.len(); + + // Our bitfield has `Lsb0`. + let msb = candidate_bitfield_bits - 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE + || candidate_bitfield_bits > MAX_BITFIELD_SIZE + // Ensure minimum bitfield size - MSB needs to be one. + || !candidate_bitfield.bit_at(msb.as_bit_index()) + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert, candidate_bitfield)) + } + } + + sanitized_assignments + } } // This adjusts the required routing of messages in blocks that pass the block filter @@ -1753,14 +2022,14 @@ impl State { #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] async fn adjust_required_routing_and_propagate( ctx: &mut Context, - peer_data: &HashMap, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: BlockFilter, routing_modifier: RoutingModifier, + peer_views: &HashMap, ) where BlockFilter: Fn(&mut BlockEntry) -> bool, - RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), + RoutingModifier: Fn(&RequiredRouting, bool, &ValidatorIndex) -> RequiredRouting, { let mut peer_assignments = HashMap::new(); let mut peer_approvals = HashMap::new(); @@ -1772,64 +2041,55 @@ async fn adjust_required_routing_and_propagate t, + None => continue, + }; - if message_state.required_routing.is_empty() { - continue - } + // We just need to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in block_entry.approval_entries.values_mut() { + let new_required_routing = routing_modifier( + &approval_entry.routing_info().required_routing, + approval_entry.routing_info().local, + &approval_entry.validator_index(), + ); - let topology = match topologies.get_topology(block_entry.session) { - Some(t) => t, - None => continue, - }; + approval_entry.update_required_routing(new_required_routing); - // Propagate the message to all peers in the required routing set. - let message_subject = MessageSubject(*block_hash, candidate_index, *validator); + if approval_entry.routing_info().required_routing.is_empty() { + continue + } - let assignment_message = ( - IndirectAssignmentCert { - block_hash: *block_hash, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: *block_hash, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(*block_hash); for (peer, peer_knowledge) in &mut block_entry.known_by { if !topology .local_grid_neighbors() - .route_to_peer(message_state.required_routing, peer) + .route_to_peer(approval_entry.routing_info().required_routing, peer) { continue } - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge.clone(), message_kind); peer_assignments .entry(*peer) .or_insert_with(Vec::new) .push(assignment_message.clone()); } - if let Some(approval_message) = approval_message.as_ref() { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in &approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(*block_hash, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); peer_approvals .entry(*peer) .or_insert_with(Vec::new) @@ -1841,24 +2101,32 @@ async fn adjust_required_routing_and_propagate continue, - Some(v) => v, - }; - - send_assignments_batched(ctx.sender(), assignments_packet, peer, peer_protocol_version) + if let Some(peer_view) = peer_views.get(&peer) { + send_assignments_batched( + ctx.sender(), + assignments_packet, + &vec![(peer, peer_view.version)], + ) .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } for (peer, approvals_packet) in peer_approvals { - let peer_protocol_version = match peer_data.get(&peer).map(|pd| pd.version) { - None => continue, - Some(v) => v, - }; - - send_approvals_batched(ctx.sender(), approvals_packet, peer, peer_protocol_version).await; + if let Some(peer_view) = peer_views.get(&peer) { + send_approvals_batched( + ctx.sender(), + approvals_packet, + &vec![(peer, peer_view.version)], + ) + .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } } @@ -1960,12 +2228,21 @@ impl ApprovalDistribution { ApprovalDistributionMessage::NewBlocks(metas) => { state.handle_new_blocks(ctx, metrics, metas, rng).await; }, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index) => { + ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { + let _span = state + .spans + .get(&cert.block_hash) + .map(|span| span.child("import-and-distribute-assignment")) + .unwrap_or_else(|| jaeger::Span::new(&cert.block_hash, "distribute-assignment")) + .with_string_tag("block-hash", format!("{:?}", cert.block_hash)) + .with_stage(jaeger::Stage::ApprovalDistribution); + gum::debug!( target: LOG_TARGET, - "Distributing our assignment on candidate (block={}, index={})", - cert.block_hash, - candidate_index, + ?candidate_indices, + block_hash = ?cert.block_hash, + assignment_kind = ?cert.cert.kind, + "Distributing our assignment on candidates", ); state @@ -1974,7 +2251,7 @@ impl ApprovalDistribution { &metrics, MessageSource::Local, cert, - candidate_index, + candidate_indices, rng, ) .await; @@ -2008,49 +2285,6 @@ impl ApprovalDistribution { } } -fn versioned_approvals_packet( - version: ValidationVersion, - approvals: Vec, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), - )), - } -} - -fn versioned_assignments_packet( - version: ValidationVersion, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments), - )), - } -} - -fn filter_peers_by_version( - peers: &[(PeerId, ValidationVersion)], - version: ValidationVersion, -) -> Vec { - peers - .iter() - .filter(|(_, v)| v == &version) - .map(|(peer_id, _)| *peer_id) - .collect() -} - #[overseer::subsystem(ApprovalDistribution, error=SubsystemError, prefix=self::overseer)] impl ApprovalDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -2075,7 +2309,7 @@ const fn ensure_size_not_zero(size: usize) -> usize { /// the protocol configuration. pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / - std::mem::size_of::<(IndirectAssignmentCert, CandidateIndex)>() / + std::mem::size_of::<(IndirectAssignmentCertV2, CandidateIndex)>() / 3, ); @@ -2084,6 +2318,54 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); +// Low level helper for sending assignments. +async fn send_assignments_batched_inner( + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + batch: impl IntoIterator, + peers: &[PeerId], + peer_version: ValidationVersion, +) { + let peers = peers.into_iter().cloned().collect::>(); + if peer_version == ValidationVersion::VStaging { + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments( + batch.into_iter().collect(), + ), + )), + )) + .await; + } else { + // Create a batch of v1 assignments from v2 assignments that are compatible with v1. + // `IndirectAssignmentCertV2` -> `IndirectAssignmentCert` + let batch = batch + .into_iter() + .filter_map(|(cert, candidates)| { + cert.try_into().ok().map(|cert| { + ( + cert, + // First 1 bit index is the candidate index. + candidates + .first_one() + .map(|index| index as CandidateIndex) + .expect("Assignment was checked for not being empty; qed"), + ) + }) + }) + .collect(); + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(batch), + )), + )) + .await; + } +} + /// Send assignments while honoring the `max_notification_size` of the protocol. /// /// Splitting the messages into multiple notifications allows more granular processing at the @@ -2091,37 +2373,81 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( /// of assignments and can `select!` other tasks. pub(crate) async fn send_assignments_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, - peer: PeerId, - protocol_version: ValidationVersion, + v2_assignments: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = assignments.into_iter().peekable(); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + // Older peers(v1) do not understand `AssignmentsV2` messages, so we have to filter these + // out. + let v1_assignments = v2_assignments + .clone() + .into_iter() + .filter(|(_, candidates)| candidates.count_ones() == 1); + + let mut v1_batches = v1_assignments.peekable(); + + while v1_batches.peek().is_some() { + let batch: Vec<_> = v1_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); + send_assignments_batched_inner(sender, batch, &v1_peers, ValidationVersion::V1).await; + } + } - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); - let versioned = versioned_assignments_packet(protocol_version, batch); + if !v2_peers.is_empty() { + let mut v2_batches = v2_assignments.into_iter().peekable(); - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + while v2_batches.peek().is_some() { + let batch = v2_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::VStaging) + .await; + } } } -/// Send approvals while honoring the `max_notification_size` of the protocol. +/// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: Vec, - peer: PeerId, - protocol_version: ValidationVersion, + approvals: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = approvals.into_iter().peekable(); - - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); - let versioned = versioned_approvals_packet(protocol_version, batch); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + let mut batches = approvals.clone().into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers.clone(), + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(batch), + )), + )) + .await; + } + } - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + if !v2_peers.is_empty() { + let mut batches = approvals.into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers.clone(), + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), + ), + ), + )) + .await; + } } } diff --git a/node/network/approval-distribution/src/metrics.rs b/node/network/approval-distribution/src/metrics.rs index 896866ce099a..6864259e6fdb 100644 --- a/node/network/approval-distribution/src/metrics.rs +++ b/node/network/approval-distribution/src/metrics.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; +use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; /// Approval Distribution metrics. #[derive(Default, Clone)] @@ -22,21 +23,34 @@ pub struct Metrics(Option); #[derive(Clone)] struct MetricsInner { - assignments_imported_total: prometheus::Counter, + assignments_imported_total: prometheus::CounterVec, approvals_imported_total: prometheus::Counter, unified_with_peer_total: prometheus::Counter, aggression_l1_messages_total: prometheus::Counter, aggression_l2_messages_total: prometheus::Counter, - time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, } +trait AsLabel { + fn as_label(&self) -> &str; +} + +impl AsLabel for &AssignmentCertKindV2 { + fn as_label(&self) -> &str { + match self { + AssignmentCertKindV2::RelayVRFDelay { .. } => "VRF Delay", + AssignmentCertKindV2::RelayVRFModulo { .. } => "VRF Modulo", + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => "VRF Modulo Compact", + } + } +} + impl Metrics { - pub(crate) fn on_assignment_imported(&self) { + pub(crate) fn on_assignment_imported(&self, kind: &AssignmentCertKindV2) { if let Some(metrics) = &self.0 { - metrics.assignments_imported_total.inc(); + metrics.assignments_imported_total.with_label_values(&[kind.as_label()]).inc(); } } @@ -89,9 +103,12 @@ impl MetricsTrait for Metrics { fn try_register(registry: &prometheus::Registry) -> Result { let metrics = MetricsInner { assignments_imported_total: prometheus::register( - prometheus::Counter::new( - "polkadot_parachain_assignments_imported_total", - "Number of valid assignments imported locally or from other peers.", + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_imported_total", + "Number of valid assignments imported locally or from other peers.", + ), + &["kind"], )?, registry, )?, @@ -124,10 +141,16 @@ impl MetricsTrait for Metrics { registry, )?, time_unify_with_peer: prometheus::register( - prometheus::Histogram::with_opts(prometheus::HistogramOpts::new( - "polkadot_parachain_time_unify_with_peer", - "Time spent within fn `unify_with_peer`.", - ).buckets(vec![0.000625, 0.00125,0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,]))?, + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_time_unify_with_peer", + "Time spent within fn `unify_with_peer`.", + ) + .buckets(vec![ + 0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + )?, registry, )?, time_import_pending_now_known: prometheus::register( diff --git a/node/network/approval-distribution/src/tests.rs b/node/network/approval-distribution/src/tests.rs index 1e9ae7b62007..f0c3c4f8ba64 100644 --- a/node/network/approval-distribution/src/tests.rs +++ b/node/network/approval-distribution/src/tests.rs @@ -24,20 +24,26 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ - AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, VrfOutput, VrfProof, + VrfSignature, + }, + v2::{ + AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2, + RELAY_VRF_MODULO_CONTEXT, + }, }; use polkadot_node_subsystem::messages::{ network_bridge_event, AllMessages, ApprovalCheckError, ReportPeerMessage, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt as _}; -use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, HashT}; +use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, CoreIndex, HashT}; use polkadot_primitives_test_helpers::dummy_signature; use rand::SeedableRng; use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; use sp_core::crypto::Pair as PairT; use std::time::Duration; - type VirtualOverseer = test_helpers::TestSubsystemContextHandle; fn test_harness>( @@ -219,15 +225,15 @@ async fn setup_gossip_topology( async fn setup_peer_with_view( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - validation_version: ValidationVersion, view: View, + version: ValidationVersion, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( *peer_id, ObservedRole::Full, - validation_version.into(), + version.into(), None, )), ) @@ -244,12 +250,28 @@ async fn setup_peer_with_view( async fn send_message_from_peer( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: protocol_v1::ApprovalDistributionMessage, +) { + overseer_send( + virtual_overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + *peer_id, + Versioned::V1(msg), + )), + ) + .await; +} + +async fn send_message_from_peer_v2( + virtual_overseer: &mut VirtualOverseer, + peer_id: &PeerId, + msg: protocol_vstaging::ApprovalDistributionMessage, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - *peer_id, msg, + *peer_id, + Versioned::VStaging(msg), )), ) .await; @@ -273,6 +295,28 @@ fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> Indirect } } +fn fake_assignment_cert_v2( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield }, + vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) }, + }, + } +} + async fn expect_reputation_change( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, @@ -331,9 +375,9 @@ fn try_import_the_same_assignment() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -353,7 +397,7 @@ fn try_import_the_same_assignment() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer_a, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_a, msg).await; expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; @@ -362,10 +406,11 @@ fn try_import_the_same_assignment() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_indices, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(claimed_indices, 0u32.into()); + assert_eq!(assignment, cert.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -385,12 +430,104 @@ fn try_import_the_same_assignment() { } ); - // setup new peer - setup_peer_with_view(overseer, &peer_d, ValidationVersion::V1, view![]).await; + // setup new peer with V2 + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_d, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_d, msg).await; + + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + +/// Just like `try_import_the_same_assignment` but use `VRFModuloCompact` assignments for multiple +/// cores. +#[test] +fn try_import_the_same_assignment_v2() { + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let peer_c = PeerId::random(); + let peer_d = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cores = vec![1, 2, 3, 4]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; + + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v2(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tx, + )) => { + assert_eq!(claimed_indices, cores.try_into().unwrap()); + assert_eq!(assignment, cert.into()); + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // setup new peer + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + + // send the same assignment from peer_d + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v2(overseer, &peer_d, msg).await; expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; @@ -413,7 +550,7 @@ fn delay_reputation_change() { let overseer = &mut virtual_overseer; // Setup peers - setup_peer_with_view(overseer, &peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -433,17 +570,18 @@ fn delay_reputation_change() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_candidates, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(assignment.cert, cert.cert.into()); + assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -474,7 +612,7 @@ fn spam_attack_results_in_negative_reputation_change() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash_b` with 20 candidates let candidates_count = 20; @@ -501,7 +639,7 @@ fn spam_attack_results_in_negative_reputation_change() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; for i in 0..candidates_count { expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; @@ -513,8 +651,8 @@ fn spam_attack_results_in_negative_reputation_change() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -533,7 +671,7 @@ fn spam_attack_results_in_negative_reputation_change() { .await; // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // each of them will incur `COST_UNEXPECTED_MESSAGE`, not only the first one for _ in 0..candidates_count { @@ -558,7 +696,7 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash` with 1 candidates let meta = BlockApprovalMeta { @@ -578,7 +716,10 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -610,12 +751,12 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { // the peer could send us it as well let assignments = vec![(cert, candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "we should not punish the peer"); // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // now we should expect_reputation_change(overseer, peer, COST_DUPLICATE_MESSAGE).await; @@ -633,10 +774,10 @@ fn import_approval_happy_path() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + // setup peers with V1 and V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -656,10 +797,14 @@ fn import_approval_happy_path() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; + // 1 peer is v1 assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( @@ -668,7 +813,21 @@ fn import_approval_happy_path() { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) )) )) => { - assert_eq!(peers.len(), 2); + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); assert_eq!(assignments.len(), 1); } ); @@ -681,7 +840,7 @@ fn import_approval_happy_path() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -722,8 +881,8 @@ fn import_approval_bad() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -749,14 +908,14 @@ fn import_approval_bad() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; // now import an assignment from peer_b let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -765,8 +924,8 @@ fn import_approval_bad() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -775,7 +934,7 @@ fn import_approval_bad() { // and try again let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -911,12 +1070,20 @@ fn update_peer_view() { let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_a, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_b, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; // connect a peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_a]).await; + setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; // we should send relevant assignments to the peer assert_matches!( @@ -934,7 +1101,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(0)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(0)); assert_eq!( state .blocks @@ -965,7 +1132,7 @@ fn update_peer_view() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_c.clone(), 0), + ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), ) .await; @@ -986,7 +1153,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(2)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(2)); assert_eq!( state .blocks @@ -1016,10 +1183,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!( - state.peer_data.get(peer).map(|data| data.view.finalized_number), - Some(finalized_number) - ); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(finalized_number)); assert!(state.blocks.get(&hash_c).unwrap().known_by.get(peer).is_none()); } @@ -1034,7 +1198,7 @@ fn import_remotely_then_locally() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup the peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1054,7 +1218,7 @@ fn import_remotely_then_locally() { let cert = fake_assignment_cert(hash, validator_index); let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( @@ -1064,8 +1228,8 @@ fn import_remotely_then_locally() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.clone().into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1075,7 +1239,10 @@ fn import_remotely_then_locally() { // import the same assignment locally overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1089,7 +1256,7 @@ fn import_remotely_then_locally() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1147,7 +1314,10 @@ fn sends_assignments_even_when_state_is_approved() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1155,7 +1325,7 @@ fn sends_assignments_even_when_state_is_approved() { .await; // connect the peer. - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1191,6 +1361,112 @@ fn sends_assignments_even_when_state_is_approved() { }); } +/// Same as `sends_assignments_even_when_state_is_approved_v2` but with `VRFModuloCompact` +/// assignemnts. +#[test] +fn sends_assignments_even_when_state_is_approved_v2() { + let peer_a = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + let peer = &peer_a; + + let _ = test_harness(State::default(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let validator_index = ValidatorIndex(0); + let cores = vec![0, 1, 2, 3]; + let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); + + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + + // Assumes candidate index == core index. + let approvals = cores + .iter() + .map(|core| IndirectSignedApprovalVote { + block_hash: hash, + candidate_index: *core, + validator: validator_index, + signature: dummy_signature(), + }) + .collect::>(); + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_bitfield.clone(), + ), + ) + .await; + + for approval in &approvals { + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ) + .await; + } + + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + + let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a + // hashmap as well. + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + /// /// /// 1. Receive remote peer view update with an unknown head @@ -1219,7 +1495,7 @@ fn race_condition_in_local_vs_remote_view_update() { }; // This will send a peer view that is ahead of our view - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_b]).await; + setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; // Send our view update to include a new head overseer_send( @@ -1240,7 +1516,7 @@ fn race_condition_in_local_vs_remote_view_update() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // This will handle pending messages being processed let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); @@ -1257,8 +1533,8 @@ fn race_condition_in_local_vs_remote_view_update() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1283,7 +1559,7 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1325,7 +1601,10 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1388,7 +1667,7 @@ fn propagates_assignments_along_unshared_dimension() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1424,7 +1703,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1473,7 +1752,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1530,7 +1809,7 @@ fn propagates_to_required_after_connect() { // Connect all peers except omitted. for (i, (peer, _)) in peers.iter().enumerate() { if !omitted.contains(&i) { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } } @@ -1573,7 +1852,10 @@ fn propagates_to_required_after_connect() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1619,7 +1901,7 @@ fn propagates_to_required_after_connect() { ); for i in omitted.iter().copied() { - setup_peer_with_view(overseer, &peers[i].0, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1).await; assert_matches!( overseer_recv(overseer).await, @@ -1668,7 +1950,7 @@ fn sends_to_more_peers_after_getting_topology() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1698,7 +1980,10 @@ fn sends_to_more_peers_after_getting_topology() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1820,7 +2105,7 @@ fn originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1857,7 +2142,10 @@ fn originator_aggression_l1() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1979,7 +2267,7 @@ fn non_originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2008,12 +2296,12 @@ fn non_originator_aggression_l1() { ) .await; - let assignments = vec![(cert.clone(), candidate_index)]; + let assignments = vec![(cert.clone().into(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2084,7 +2372,7 @@ fn non_originator_aggression_l2() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2118,7 +2406,7 @@ fn non_originator_aggression_l2() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2249,7 +2537,7 @@ fn resends_messages_periodically() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -2284,7 +2572,7 @@ fn resends_messages_periodically() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2375,126 +2663,6 @@ fn resends_messages_periodically() { }); } -/// Tests that peers correctly receive versioned messages. -#[test] -fn import_versioned_approval() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let parent_hash = Hash::repeat_byte(0xFF); - let hash = Hash::repeat_byte(0xAA); - - let state = state_without_reputation_delay(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // All peers are aware of relay parent. - setup_peer_with_view(overseer, &peer_a, ValidationVersion::VStaging, view![hash]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::VStaging, view![hash]).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), - ) - .await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(assignments.len(), 1); - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_c)); - - assert_eq!(assignments.len(), 1); - } - ); - - // send the an approval from peer_a - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_a, Versioned::VStaging(msg)).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); - - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - // Peers b and c receive versioned approval messages. - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(approvals.len(), 1); - } - ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_c]); - assert_eq!(approvals.len(), 1); - } - ); - virtual_overseer - }); -} - fn batch_test_round(message_count: usize) { use polkadot_node_subsystem::SubsystemContext; let pool = sp_core::testing::TaskExecutor::new(); @@ -2512,7 +2680,9 @@ fn batch_test_round(message_count: usize) { let validators = 0..message_count; let assignments: Vec<_> = validators .clone() - .map(|index| (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)), 0)) + .map(|index| { + (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)).into(), 0.into()) + }) .collect(); let approvals: Vec<_> = validators @@ -2525,9 +2695,18 @@ fn batch_test_round(message_count: usize) { .collect(); let peer = PeerId::random(); - send_assignments_batched(&mut sender, assignments.clone(), peer, ValidationVersion::V1) - .await; - send_approvals_batched(&mut sender, approvals.clone(), peer, ValidationVersion::V1).await; + send_assignments_batched( + &mut sender, + assignments.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; + send_approvals_batched( + &mut sender, + approvals.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; // Check expected assignments batches. for assignment_index in (0..assignments.len()).step_by(super::MAX_ASSIGNMENT_BATCH_SIZE) { @@ -2549,7 +2728,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, assignment) in sent_assignments.iter().enumerate() { - assert_eq!(assignment.0, assignments[assignment_index + message_index].0); + assert_eq!(assignment.0, assignments[assignment_index + message_index].0.clone().try_into().unwrap()); assert_eq!(assignment.1, 0); } } diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index c85d874bc4db..55243edd3a5e 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -131,9 +131,9 @@ pub struct PeerData { /// Data used to track information of peers and relay parents the /// overseer ordered us to work on. -#[derive(Default, Debug)] +#[derive(Default)] struct ProtocolState { - /// Track all active peers and their views + /// Track all active peer views and protocol versions /// to determine what is relevant to them. peer_data: HashMap, @@ -775,9 +775,11 @@ async fn handle_network_msg( handle_peer_view_change(ctx, state, new_peer, old_view, rng).await; } }, - NetworkBridgeEvent::PeerViewChange(peerid, new_view) => { - gum::trace!(target: LOG_TARGET, ?peerid, ?new_view, "Peer view change"); - handle_peer_view_change(ctx, state, peerid, new_view, rng).await; + NetworkBridgeEvent::PeerViewChange(peer_id, new_view) => { + gum::trace!(target: LOG_TARGET, ?peer_id, ?new_view, "Peer view change"); + if state.peer_data.get(&peer_id).is_some() { + handle_peer_view_change(ctx, state, peer_id, new_view, rng).await; + } }, NetworkBridgeEvent::OurViewChange(new_view) => { gum::trace!(target: LOG_TARGET, ?new_view, "Our view change"); diff --git a/node/network/bridge/src/network.rs b/node/network/bridge/src/network.rs index 4f21212dcb64..4d2d1fba4891 100644 --- a/node/network/bridge/src/network.rs +++ b/node/network/bridge/src/network.rs @@ -28,23 +28,109 @@ use sc_network::{ }; use polkadot_node_network_protocol::{ - peer_set::{PeerSet, PeerSetProtocolNames, ProtocolVersion}, + peer_set::{ + CollationVersion, PeerSet, PeerSetProtocolNames, ProtocolVersion, ValidationVersion, + }, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - PeerId, + v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; -use crate::validator_discovery::AuthorityDiscovery; +use crate::{metrics::Metrics, validator_discovery::AuthorityDiscovery, WireMessage}; // network bridge network abstraction log target const LOG_TARGET: &'static str = "parachain::network-bridge-net"; -/// Send a message to the network. +// Helper function to send a validation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v1 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a validation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +/// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. -pub(crate) fn send_message( +fn send_message( net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, @@ -64,6 +150,17 @@ pub(crate) fn send_message( encoded }; + // optimization: generate the protocol name once. + let protocol_name = protocol_names.get_name(peer_set, version); + gum::trace!( + target: LOG_TARGET, + ?peers, + ?version, + ?protocol_name, + ?message, + "Sending message to peers", + ); + // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. diff --git a/node/network/bridge/src/rx/mod.rs b/node/network/bridge/src/rx/mod.rs index 002919c5b0e5..15ed6b48745d 100644 --- a/node/network/bridge/src/rx/mod.rs +++ b/node/network/bridge/src/rx/mod.rs @@ -64,9 +64,11 @@ use super::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; - -use crate::network::get_peer_id_by_authority_id; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; +use crate::{network::get_peer_id_by_authority_id, WireMessage}; use super::metrics::Metrics; @@ -250,22 +252,18 @@ where match ValidationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - ValidationVersion::V1 => send_message( + ValidationVersion::V1 => send_validation_message_v1( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - ValidationVersion::VStaging => send_message( + ValidationVersion::VStaging => send_validation_message_v2( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -292,22 +290,18 @@ where match CollationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - CollationVersion::V1 => send_message( + CollationVersion::V1 => send_collation_message_v1( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - CollationVersion::VStaging => send_message( + CollationVersion::VStaging => send_collation_message_v2( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -384,8 +378,16 @@ where .filter_map(|(protocol, msg_bytes)| { // version doesn't matter because we always receive on the 'correct' // protocol name, not the negotiated fallback. - let (peer_set, _version) = + let (peer_set, version) = peerset_protocol_names.try_get_protocol(protocol)?; + gum::trace!( + target: LOG_TARGET, + ?peer_set, + ?protocol, + ?version, + "Received notification" + ); + if peer_set == PeerSet::Validation { if expected_versions[PeerSet::Validation].is_none() { return Some(Err(UNCONNECTED_PEERSET_COST)) @@ -832,7 +834,7 @@ fn update_our_view( metrics, ); - send_validation_message_vstaging( + send_validation_message_v2( net, vstaging_validation_peers, peerset_protocol_names, @@ -840,7 +842,7 @@ fn update_our_view( metrics, ); - send_collation_message_vstaging( + send_collation_message_v2( net, vstaging_collation_peers, peerset_protocol_names, @@ -917,78 +919,6 @@ fn handle_peer_messages>( (outgoing_events, reports) } -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, diff --git a/node/network/bridge/src/tx/mod.rs b/node/network/bridge/src/tx/mod.rs index e0ca633547f4..7bd7f1e25c06 100644 --- a/node/network/bridge/src/tx/mod.rs +++ b/node/network/bridge/src/tx/mod.rs @@ -18,9 +18,7 @@ use super::*; use polkadot_node_network_protocol::{ - peer_set::{CollationVersion, PeerSet, PeerSetProtocolNames, ValidationVersion}, - request_response::ReqProtocolNames, - v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, Versioned, + peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, Versioned, }; use polkadot_node_subsystem::{ @@ -40,7 +38,10 @@ use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; use crate::metrics::Metrics; @@ -186,6 +187,7 @@ where gum::trace!( target: LOG_TARGET, action = "SendValidationMessages", + ?msg, num_messages = 1usize, ); @@ -197,7 +199,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -211,6 +213,7 @@ where target: LOG_TARGET, action = "SendValidationMessages", num_messages = %msgs.len(), + ?msgs, ); for (peers, msg) in msgs { @@ -222,7 +225,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -247,7 +250,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -272,7 +275,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -373,75 +376,3 @@ where Ok(()) } - -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} diff --git a/node/network/bridge/src/tx/tests.rs b/node/network/bridge/src/tx/tests.rs index 21cd134c54f2..6e28a4032183 100644 --- a/node/network/bridge/src/tx/tests.rs +++ b/node/network/bridge/src/tx/tests.rs @@ -25,9 +25,9 @@ use std::collections::HashSet; use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSetProtocolNames, ValidationVersion}, request_response::{outgoing::Requests, ReqProtocolNames}, - ObservedRole, Versioned, + v1 as protocol_v1, vstaging as protocol_vstaging, ObservedRole, Versioned, }; use polkadot_node_subsystem::{FromOrchestra, OverseerSignal}; use polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; @@ -356,7 +356,6 @@ fn network_protocol_versioning_send() { } // send a validation protocol message. - { let approval_distribution_message = protocol_vstaging::ApprovalDistributionMessage::Approvals(Vec::new()); diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index 1bed2c12fe20..ba9b9a7f4900 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -313,7 +313,6 @@ macro_rules! impl_versioned_full_protocol_from { } }; } - /// Implement `TryFrom` for one versioned enum variant into the inner type. /// `$m_ty::$variant(inner) -> Ok(inner)` macro_rules! impl_versioned_try_from { @@ -441,7 +440,7 @@ pub mod v1 { }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::v1::{IndirectAssignmentCert, IndirectSignedApprovalVote}, UncheckedSignedFullStatement, }; @@ -595,12 +594,15 @@ pub mod vstaging { use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::vstaging::{ - CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, - Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, + UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::IndirectSignedApprovalVote, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, UncheckedSignedFullStatement, }; @@ -769,10 +771,13 @@ pub mod vstaging { #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub enum ApprovalDistributionMessage { /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to + /// candidate index. /// /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. #[codec(index = 0)] - Assignments(Vec<(IndirectAssignmentCert, CandidateIndex)>), + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), /// Approvals for candidates in some recent, unfinalized block. #[codec(index = 1)] Approvals(Vec), @@ -841,3 +846,11 @@ pub mod vstaging { payload } } + +/// Returns the subset of `peers` with the specified `version`. +pub fn filter_by_peer_version( + peers: &[(PeerId, peer_set::ProtocolVersion)], + version: peer_set::ProtocolVersion, +) -> Vec { + peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() +} diff --git a/node/primitives/Cargo.toml b/node/primitives/Cargo.toml index fbd85c1a0551..a8381916ffd6 100644 --- a/node/primitives/Cargo.toml +++ b/node/primitives/Cargo.toml @@ -21,6 +21,7 @@ polkadot-parachain = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" thiserror = "1.0.31" serde = { version = "1.0.163", features = ["derive"] } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] zstd = { version = "0.11.2", default-features = false } diff --git a/node/primitives/src/approval.rs b/node/primitives/src/approval.rs index 01f45a207874..bfa1f1aa47d2 100644 --- a/node/primitives/src/approval.rs +++ b/node/primitives/src/approval.rs @@ -16,190 +16,515 @@ //! Types relevant for approval. -pub use sp_consensus_babe::{Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript}; - -use parity_scale_codec::{Decode, Encode}; -use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, - ValidatorIndex, ValidatorSignature, -}; -use sp_application_crypto::ByteArray; -use sp_consensus_babe as babe_primitives; - -/// Validators assigning to check a particular candidate are split up into tranches. -/// Earlier tranches of validators check first, with later tranches serving as backup. -pub type DelayTranche = u32; - -/// A static context used to compute the Relay VRF story based on the -/// VRF output included in the header-chain. -pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; - -/// A static context used for transcripts indicating assigned availability core. -pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; - -/// A static context associated with producing randomness for a core. -pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; - -/// A static context associated with producing randomness for a tranche. -pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; - -/// random bytes derived from the VRF submitted within the block by the -/// block author as a credential and used as input to approval assignment criteria. -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -pub struct RelayVRFStory(pub [u8; 32]); - -/// Different kinds of input data or criteria that can prove a validator's assignment -/// to check a particular parachain. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub enum AssignmentCertKind { - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with a sample number. +/// A list of primitives introduced in v1. +pub mod v1 { + use sp_consensus_babe as babe_primitives; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + + use parity_scale_codec::{Decode, Encode}; + use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, + ValidatorIndex, ValidatorSignature, + }; + use sp_application_crypto::ByteArray; + + /// Validators assigning to check a particular candidate are split up into tranches. + /// Earlier tranches of validators check first, with later tranches serving as backup. + pub type DelayTranche = u32; + + /// A static context used to compute the Relay VRF story based on the + /// VRF output included in the header-chain. + pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; + + /// A static context used for transcripts indicating assigned availability core. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; + + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; + + /// A static context associated with producing randomness for a tranche. + pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; + + /// random bytes derived from the VRF submitted within the block by the + /// block author as a credential and used as input to approval assignment criteria. + #[derive(Debug, Clone, Encode, Decode, PartialEq)] + pub struct RelayVRFStory(pub [u8; 32]); + + /// Different kinds of input data or criteria that can prove a validator's assignment + /// to check a particular parachain. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKind { + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCert { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKind, + /// The VRF signature showing the criterion is met. + pub vrf: VrfSignature, + } + + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCert { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCert, + } + + /// A signed approval vote which references the candidate indirectly via the block. /// - /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] - RelayVRFModulo { - /// The sample number used in this cert. - sample: u32, - }, - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with the index of a particular core. + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVote { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_index: CandidateIndex, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } + + /// Metadata about a block which is now live in the approval protocol. + #[derive(Debug)] + pub struct BlockApprovalMeta { + /// The hash of the block. + pub hash: Hash, + /// The number of the block. + pub number: BlockNumber, + /// The hash of the parent block. + pub parent_hash: Hash, + /// The candidates included by the block. + /// Note that these are not the same as the candidates that appear within the block body. + pub candidates: Vec, + /// The consensus slot of the block. + pub slot: Slot, + /// The session of the block. + pub session: SessionIndex, + } + + /// Errors that can occur during the approvals protocol. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum ApprovalError { + #[error("Schnorrkel signature error")] + SchnorrkelSignature(schnorrkel::errors::SignatureError), + #[error("Authority index {0} out of bounds")] + AuthorityOutOfBounds(usize), + } + + /// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. + pub struct UnsafeVRFOutput { + vrf_output: VrfOutput, + slot: Slot, + authority_index: u32, + } + + impl UnsafeVRFOutput { + /// Get the slot. + pub fn slot(&self) -> Slot { + self.slot + } + + /// Compute the randomness associated with this VRF output. + pub fn compute_randomness( + self, + authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], + randomness: &babe_primitives::Randomness, + epoch_index: u64, + ) -> Result { + let author = match authorities.get(self.authority_index as usize) { + None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), + Some(x) => &x.0, + }; + + let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) + .map_err(ApprovalError::SchnorrkelSignature)?; + + let transcript = + sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + + let inout = self + .vrf_output + .0 + .attach_input_hash(&pubkey, transcript.0) + .map_err(ApprovalError::SchnorrkelSignature)?; + Ok(RelayVRFStory(inout.make_bytes(super::v1::RELAY_VRF_STORY_CONTEXT))) + } + } + + /// Extract the slot number and relay VRF from a header. /// - /// The context is [`RELAY_VRF_DELAY_CONTEXT`] - RelayVRFDelay { - /// The core index chosen in this cert. - core_index: CoreIndex, - }, -} + /// This fails if either there is no BABE `PreRuntime` digest or + /// the digest has type `SecondaryPlain`, which Substrate nodes do + /// not produce or accept anymore. + pub fn babe_unsafe_vrf_info(header: &Header) -> Option { + use babe_primitives::digests::CompatibleDigestItem; -/// A certification of assignment. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct AssignmentCert { - /// The criterion which is claimed to be met by this cert. - pub kind: AssignmentCertKind, - /// The VRF signature showing the criterion is met. - pub vrf: VrfSignature, -} + for digest in &header.digest.logs { + if let Some(pre) = digest.as_babe_pre_digest() { + let slot = pre.slot(); + let authority_index = pre.authority_index(); -/// An assignment criterion which refers to the candidate under which the assignment is -/// relevant by block hash. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectAssignmentCert { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The validator index. - pub validator: ValidatorIndex, - /// The cert itself. - pub cert: AssignmentCert, -} + return pre.vrf_signature().map(|sig| UnsafeVRFOutput { + vrf_output: sig.output.clone(), + slot, + authority_index, + }) + } + } -/// A signed approval vote which references the candidate indirectly via the block. -/// -/// In practice, we have a look-up from block hash and candidate index to candidate hash, -/// so this can be transformed into a `SignedApprovalVote`. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectSignedApprovalVote { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The index of the candidate in the list of candidates fully included as-of the block. - pub candidate_index: CandidateIndex, - /// The validator index. - pub validator: ValidatorIndex, - /// The signature by the validator. - pub signature: ValidatorSignature, + None + } } -/// Metadata about a block which is now live in the approval protocol. -#[derive(Debug)] -pub struct BlockApprovalMeta { - /// The hash of the block. - pub hash: Hash, - /// The number of the block. - pub number: BlockNumber, - /// The hash of the parent block. - pub parent_hash: Hash, - /// The candidates included by the block. - /// Note that these are not the same as the candidates that appear within the block body. - pub candidates: Vec, - /// The consensus slot of the block. - pub slot: Slot, - /// The session of the block. - pub session: SessionIndex, -} +/// A list of primitives introduced by v2. +pub mod v2 { + use parity_scale_codec::{Decode, Encode}; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + use std::ops::BitOr; -/// Errors that can occur during the approvals protocol. -#[derive(Debug, thiserror::Error)] -#[allow(missing_docs)] -pub enum ApprovalError { - #[error("Schnorrkel signature error")] - SchnorrkelSignature(schnorrkel::errors::SignatureError), - #[error("Authority index {0} out of bounds")] - AuthorityOutOfBounds(usize), -} + use bitvec::{prelude::Lsb0, vec::BitVec}; + use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex}; -/// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. -pub struct UnsafeVRFOutput { - vrf_output: VrfOutput, - slot: Slot, - authority_index: u32, -} + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; + /// A static context associated with producing randomness for v2 multi-core assignments. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED v2"; + /// A static context used for all relay-vrf-modulo VRFs for v2 multi-core assignments. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD v2"; + /// A read-only bitvec wrapper + #[derive(Clone, Debug, Encode, Decode, Hash, PartialEq, Eq)] + pub struct Bitfield(BitVec, std::marker::PhantomData); + + /// A `read-only`, `non-zero` bitfield. + /// Each 1 bit identifies a candidate by the bitfield bit index. + pub type CandidateBitfield = Bitfield; + /// A bitfield of core assignments. + pub type CoreBitfield = Bitfield; + + /// Errors that can occur when creating and manipulating bitfields. + #[derive(Debug)] + pub enum BitfieldError { + /// All bits are zero. + NullAssignment, + } + + /// A bit index in `Bitfield`. + #[cfg_attr(test, derive(PartialEq, Clone))] + pub struct BitIndex(pub usize); + + /// Helper trait to convert primitives to `BitIndex`. + pub trait AsBitIndex { + /// Returns the index of the corresponding bit in `Bitfield`. + fn as_bit_index(&self) -> BitIndex; + } + + impl Bitfield { + /// Returns the bit value at specified `index`. If `index` is greater than bitfield size, + /// returns `false`. + pub fn bit_at(&self, index: BitIndex) -> bool { + if self.0.len() <= index.0 { + false + } else { + self.0[index.0] + } + } + + /// Returns number of bits. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns the number of 1 bits. + pub fn count_ones(&self) -> usize { + self.0.count_ones() + } + + /// Returns the index of the first 1 bit. + pub fn first_one(&self) -> Option { + self.0.first_one() + } + + /// Returns an iterator over inner bits. + pub fn iter_ones(&self) -> bitvec::slice::IterOnes { + self.0.iter_ones() + } -impl UnsafeVRFOutput { - /// Get the slot. - pub fn slot(&self) -> Slot { - self.slot + /// For testing purpose, we want a inner mutable ref. + #[cfg(test)] + pub fn inner_mut(&mut self) -> &mut BitVec { + &mut self.0 + } + + /// Returns the inner bitfield and consumes `self`. + pub fn into_inner(self) -> BitVec { + self.0 + } } - /// Compute the randomness associated with this VRF output. - pub fn compute_randomness( - self, - authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], - randomness: &babe_primitives::Randomness, - epoch_index: u64, - ) -> Result { - let author = match authorities.get(self.authority_index as usize) { - None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), - Some(x) => &x.0, - }; + impl AsBitIndex for CandidateIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self as usize) + } + } - let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) - .map_err(ApprovalError::SchnorrkelSignature)?; + impl AsBitIndex for CoreIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(self.0 as usize) + } + } - let transcript = sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + impl AsBitIndex for usize { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self) + } + } - let inout = self - .vrf_output - .0 - .attach_input_hash(&pubkey, transcript.0) - .map_err(ApprovalError::SchnorrkelSignature)?; - Ok(RelayVRFStory(inout.make_bytes(RELAY_VRF_STORY_CONTEXT))) + impl From for Bitfield + where + T: AsBitIndex, + { + fn from(value: T) -> Self { + Self( + { + let mut bv = bitvec::bitvec![u8, Lsb0; 0; value.as_bit_index().0 + 1]; + bv.set(value.as_bit_index().0, true); + bv + }, + Default::default(), + ) + } + } + + impl TryFrom> for Bitfield + where + T: Into>, + { + type Error = BitfieldError; + + fn try_from(mut value: Vec) -> Result { + if value.is_empty() { + return Err(BitfieldError::NullAssignment) + } + + let initial_bitfield = + value.pop().expect("Just checked above it's not empty; qed").into(); + + Ok(Self( + value.into_iter().fold(initial_bitfield.0, |initial_bitfield, element| { + let mut bitfield: Bitfield = element.into(); + bitfield + .0 + .resize(std::cmp::max(initial_bitfield.len(), bitfield.0.len()), false); + bitfield.0.bitor(initial_bitfield) + }), + Default::default(), + )) + } + } + + /// Certificate is changed compared to `AssignmentCertKind`: + /// - introduced RelayVRFModuloCompact + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block + /// where the candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 0)] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + #[codec(index = 1)] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 2)] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCertV2 { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKindV2, + /// The VRF showing the criterion is met. + pub vrf: VrfSignature, + } + + impl From for AssignmentCertV2 { + fn from(cert: super::v1::AssignmentCert) -> Self { + Self { + kind: match cert.kind { + super::v1::AssignmentCertKind::RelayVRFDelay { core_index } => + AssignmentCertKindV2::RelayVRFDelay { core_index }, + super::v1::AssignmentCertKind::RelayVRFModulo { sample } => + AssignmentCertKindV2::RelayVRFModulo { sample }, + }, + vrf: cert.vrf, + } + } } -} -/// Extract the slot number and relay VRF from a header. -/// -/// This fails if either there is no BABE `PreRuntime` digest or -/// the digest has type `SecondaryPlain`, which Substrate nodes do -/// not produce or accept anymore. -pub fn babe_unsafe_vrf_info(header: &Header) -> Option { - use babe_primitives::digests::CompatibleDigestItem; - - for digest in &header.digest.logs { - if let Some(pre) = digest.as_babe_pre_digest() { - let slot = pre.slot(); - let authority_index = pre.authority_index(); - - return pre.vrf_signature().map(|sig| UnsafeVRFOutput { - vrf_output: sig.output.clone(), - slot, - authority_index, + /// Errors that can occur when trying to convert to/from assignment v1/v2 + #[derive(Debug)] + pub enum AssignmentConversionError { + /// Assignment certificate is not supported in v1. + CertificateNotSupported, + } + + impl TryFrom for super::v1::AssignmentCert { + type Error = AssignmentConversionError; + fn try_from(cert: AssignmentCertV2) -> Result { + Ok(Self { + kind: match cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => + super::v1::AssignmentCertKind::RelayVRFDelay { core_index }, + AssignmentCertKindV2::RelayVRFModulo { sample } => + super::v1::AssignmentCertKind::RelayVRFModulo { sample }, + // Not supported + _ => return Err(AssignmentConversionError::CertificateNotSupported), + }, + vrf: cert.vrf, }) } } - None + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCertV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCertV2, + } + + impl From for IndirectAssignmentCertV2 { + fn from(indirect_cert: super::v1::IndirectAssignmentCert) -> Self { + Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.into(), + } + } + } + + impl TryFrom for super::v1::IndirectAssignmentCert { + type Error = AssignmentConversionError; + fn try_from( + indirect_cert: IndirectAssignmentCertV2, + ) -> Result { + Ok(Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.try_into()?, + }) + } + } +} + +#[cfg(test)] +mod test { + use super::v2::{BitIndex, Bitfield}; + + use polkadot_primitives::{CandidateIndex, CoreIndex}; + + #[test] + fn test_assignment_bitfield_from_vec() { + let candidate_indices = vec![1u32, 7, 3, 10, 45, 8, 200, 2]; + let max_index = *candidate_indices.iter().max().unwrap(); + let bitfield = Bitfield::try_from(candidate_indices.clone()).unwrap(); + let candidate_indices = + candidate_indices.into_iter().map(|i| BitIndex(i as usize)).collect::>(); + + // Test 1 bits. + for index in candidate_indices.clone() { + assert!(bitfield.bit_at(index)); + } + + // Test 0 bits. + for index in 0..max_index { + if candidate_indices.contains(&BitIndex(index as usize)) { + continue + } + assert!(!bitfield.bit_at(BitIndex(index as usize))); + } + } + + #[test] + fn test_assignment_bitfield_invariant_msb() { + let core_indices = vec![CoreIndex(1), CoreIndex(3), CoreIndex(10), CoreIndex(20)]; + let mut bitfield = Bitfield::try_from(core_indices.clone()).unwrap(); + assert!(bitfield.inner_mut().pop().unwrap()); + + for i in 0..1024 { + assert!(Bitfield::try_from(CoreIndex(i)).unwrap().inner_mut().pop().unwrap()); + assert!(Bitfield::try_from(i).unwrap().inner_mut().pop().unwrap()); + } + } + + #[test] + fn test_assignment_bitfield_basic() { + let bitfield = Bitfield::try_from(CoreIndex(0)).unwrap(); + assert!(bitfield.bit_at(BitIndex(0))); + assert!(!bitfield.bit_at(BitIndex(1))); + assert_eq!(bitfield.len(), 1); + + let mut bitfield = Bitfield::try_from(20 as CandidateIndex).unwrap(); + assert!(bitfield.bit_at(BitIndex(20))); + assert_eq!(bitfield.inner_mut().count_ones(), 1); + assert_eq!(bitfield.len(), 21); + } } diff --git a/node/service/src/parachains_db/mod.rs b/node/service/src/parachains_db/mod.rs index 519afbe0ccd1..92f3f167f22f 100644 --- a/node/service/src/parachains_db/mod.rs +++ b/node/service/src/parachains_db/mod.rs @@ -41,7 +41,15 @@ pub(crate) mod columns { pub const COL_SESSION_WINDOW_DATA: u32 = 5; } + // Version 4 only changed structures in approval voting, so we can re-export the v4 definitions. pub mod v3 { + pub use super::v4::{ + COL_APPROVAL_DATA, COL_AVAILABILITY_DATA, COL_AVAILABILITY_META, + COL_CHAIN_SELECTION_DATA, COL_DISPUTE_COORDINATOR_DATA, NUM_COLUMNS, ORDERED_COL, + }; + } + + pub mod v4 { pub const NUM_COLUMNS: u32 = 5; pub const COL_AVAILABILITY_DATA: u32 = 0; pub const COL_AVAILABILITY_META: u32 = 1; @@ -73,14 +81,14 @@ pub struct ColumnsConfig { /// The real columns used by the parachains DB. #[cfg(any(test, feature = "full-node"))] pub const REAL_COLUMNS: ColumnsConfig = ColumnsConfig { - col_availability_data: columns::v3::COL_AVAILABILITY_DATA, - col_availability_meta: columns::v3::COL_AVAILABILITY_META, - col_approval_data: columns::v3::COL_APPROVAL_DATA, - col_chain_selection_data: columns::v3::COL_CHAIN_SELECTION_DATA, - col_dispute_coordinator_data: columns::v3::COL_DISPUTE_COORDINATOR_DATA, + col_availability_data: columns::v4::COL_AVAILABILITY_DATA, + col_availability_meta: columns::v4::COL_AVAILABILITY_META, + col_approval_data: columns::v4::COL_APPROVAL_DATA, + col_chain_selection_data: columns::v4::COL_CHAIN_SELECTION_DATA, + col_dispute_coordinator_data: columns::v4::COL_DISPUTE_COORDINATOR_DATA, }; -#[derive(PartialEq)] +#[derive(PartialEq, Copy, Clone)] pub(crate) enum DatabaseKind { ParityDB, RocksDB, @@ -125,28 +133,28 @@ pub fn open_creating_rocksdb( let path = root.join("parachains").join("db"); - let mut db_config = DatabaseConfig::with_columns(columns::v3::NUM_COLUMNS); + let mut db_config = DatabaseConfig::with_columns(columns::v4::NUM_COLUMNS); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_DATA, cache_sizes.availability_data); + .insert(columns::v4::COL_AVAILABILITY_DATA, cache_sizes.availability_data); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_META, cache_sizes.availability_meta); + .insert(columns::v4::COL_AVAILABILITY_META, cache_sizes.availability_meta); let _ = db_config .memory_budget - .insert(columns::v3::COL_APPROVAL_DATA, cache_sizes.approval_data); + .insert(columns::v4::COL_APPROVAL_DATA, cache_sizes.approval_data); let path_str = path .to_str() .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB, upgrade::CURRENT_VERSION)?; let db = Database::open(&db_config, &path_str)?; let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) @@ -164,14 +172,14 @@ pub fn open_creating_paritydb( .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB, upgrade::CURRENT_VERSION)?; let db = parity_db::Db::open_or_create(&upgrade::paritydb_version_3_config(&path)) .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?; let db = polkadot_node_subsystem_util::database::paritydb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) } diff --git a/node/service/src/parachains_db/upgrade.rs b/node/service/src/parachains_db/upgrade.rs index 54ef97afd71c..b99f885176b0 100644 --- a/node/service/src/parachains_db/upgrade.rs +++ b/node/service/src/parachains_db/upgrade.rs @@ -22,13 +22,17 @@ use std::{ str::FromStr, }; +use polkadot_node_core_approval_voting::approval_db::v2::{ + migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +}; type Version = u32; /// Version file name. const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. -const CURRENT_VERSION: Version = 3; +/// Version 4 changes approval db format for `OurAssignment`. +pub(crate) const CURRENT_VERSION: Version = 4; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -38,6 +42,10 @@ pub enum Error { CorruptedVersionFile, #[error("Parachains DB has a future version (expected {current:?}, found {got:?})")] FutureVersion { current: Version, got: Version }, + #[error("Parachain DB migration failed")] + MigrationFailed, + #[error("Parachain DB migration would take forever")] + MigrationLoop, } impl From for io::Error { @@ -49,10 +57,42 @@ impl From for io::Error { } } -/// Try upgrading parachain's database to the current version. -pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +/// Try upgrading parachain's database to a target version. +pub(crate) fn try_upgrade_db( + db_path: &Path, + db_kind: DatabaseKind, + target_version: Version, +) -> Result<(), Error> { + // Ensure we don't loop forever below befcause of a bug. + const MAX_MIGRATIONS: u32 = 30; + + #[cfg(test)] + remove_file_lock(&db_path); + + // Loop migrations until we reach the target version. + for _ in 0..MAX_MIGRATIONS { + let version = try_upgrade_db_to_next_version(db_path, db_kind)?; + + #[cfg(test)] + remove_file_lock(&db_path); + + if version == target_version { + return Ok(()) + } + } + + Err(Error::MigrationLoop) +} + +/// Try upgrading parachain's database to the next version. +/// If successfull, it returns the current version. +pub(crate) fn try_upgrade_db_to_next_version( + db_path: &Path, + db_kind: DatabaseKind, +) -> Result { let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); - if !is_empty { + + let new_version = if !is_empty { match get_db_version(db_path)? { // 0 -> 1 migration Some(0) => migrate_from_version_0_to_1(db_path, db_kind)?, @@ -60,21 +100,26 @@ pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<() Some(1) => migrate_from_version_1_to_2(db_path, db_kind)?, // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, + // 3 -> 4 migration + Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, // Already at current version, do nothing. - Some(CURRENT_VERSION) => (), + Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. Some(v) => return Err(Error::FutureVersion { current: CURRENT_VERSION, got: v }), // No version file. For `RocksDB` we dont need to do anything. - None if db_kind == DatabaseKind::RocksDB => (), + None if db_kind == DatabaseKind::RocksDB => CURRENT_VERSION, // No version file. `ParityDB` did not previously have a version defined. // We handle this as a `0 -> 1` migration. None if db_kind == DatabaseKind::ParityDB => migrate_from_version_0_to_1(db_path, db_kind)?, None => unreachable!(), } - } + } else { + CURRENT_VERSION + }; - update_version(db_path) + update_version(db_path, new_version)?; + Ok(new_version) } /// Reads current database version from the file at given path. @@ -91,9 +136,9 @@ fn get_db_version(path: &Path) -> Result, Error> { /// Writes current database version to the file. /// Creates a new file if the version file does not exist yet. -fn update_version(path: &Path) -> Result<(), Error> { +fn update_version(path: &Path, new_version: Version) -> Result<(), Error> { fs::create_dir_all(path)?; - fs::write(version_file_path(path), CURRENT_VERSION.to_string()).map_err(Into::into) + fs::write(version_file_path(path), new_version.to_string()).map_err(Into::into) } /// Returns the version file path. @@ -103,7 +148,7 @@ fn version_file_path(path: &Path) -> PathBuf { file_path } -fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 0 to version 1 ..."); match db_kind { @@ -116,7 +161,7 @@ fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 1 to version 2 ..."); match db_kind { @@ -129,7 +174,48 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +// Migrade approval voting database. `OurAssignment` has been changed to support the v2 assignments. +// As these are backwards compatible, we'll convert the old entries in the new format. +fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { + gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); + use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, + }; + use std::sync::Arc; + + let approval_db_config = + ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; + + let _result = match db_kind { + DatabaseKind::ParityDB => { + let db = ParityDbAdapter::new( + parity_db::Db::open(&paritydb_version_3_config(path)) + .map_err(|e| other_io_error(format!("Error opening db {:?}", e)))?, + super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + DatabaseKind::RocksDB => { + let db_path = path + .to_str() + .ok_or_else(|| super::other_io_error("Invalid database path".into()))?; + let db_cfg = + kvdb_rocksdb::DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + let db = RocksDbAdapter::new( + kvdb_rocksdb::Database::open(&db_cfg, db_path)?, + &super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + }; + + gum::info!(target: LOG_TARGET, "Migration complete! "); + Ok(CURRENT_VERSION) +} + +fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 2 to version 3 ..."); match db_kind { DatabaseKind::ParityDB => paritydb_migrate_from_version_2_to_3(path), @@ -143,7 +229,7 @@ fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), /// Migration from version 0 to version 1: /// * the number of columns has changed from 3 to 5; -fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -155,12 +241,12 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { db.add_column()?; db.add_column()?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// * the number of columns has changed from 5 to 6; -fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -171,10 +257,10 @@ fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { db.add_column()?; - Ok(()) + Ok(2) } -fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -185,7 +271,7 @@ fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { db.remove_last_column()?; - Ok(()) + Ok(3) } // This currently clears columns which had their configs altered between versions. @@ -249,7 +335,7 @@ fn paritydb_fix_columns( pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v1::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -260,7 +346,7 @@ pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { pub(crate) fn paritydb_version_2_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v2::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -278,48 +364,160 @@ pub(crate) fn paritydb_version_3_config(path: &Path) -> parity_db::Options { options } +/// Database configuration for version 0. This is useful just for testing. +#[cfg(test)] +pub(crate) fn paritydb_version_0_config(path: &Path) -> parity_db::Options { + let mut options = + parity_db::Options::with_columns(&path, super::columns::v0::NUM_COLUMNS as u8); + options.columns[super::columns::v4::COL_AVAILABILITY_META as usize].btree_index = true; + + options +} + /// Migration from version 0 to version 1. /// Cases covered: /// - upgrading from v0.9.23 or earlier -> the `dispute coordinator column` was changed /// - upgrading from v0.9.24+ -> this is a no op assuming the DB has been manually fixed as per /// release notes -fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result { // Delete the `dispute coordinator` column if needed (if column configuration is changed). paritydb_fix_columns( path, paritydb_version_1_config(path), - vec![super::columns::v3::COL_DISPUTE_COORDINATOR_DATA], + vec![super::columns::v4::COL_DISPUTE_COORDINATOR_DATA], )?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// - add a new column for session information storage -fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result { let mut options = paritydb_version_1_config(path); // Adds the session info column. parity_db::Db::add_column(&mut options, Default::default()) .map_err(|e| other_io_error(format!("Error adding column {:?}", e)))?; - Ok(()) + Ok(2) } /// Migration from version 2 to version 3: /// - drop the column used by `RollingSessionWindow` -fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result { parity_db::Db::drop_last_column(&mut paritydb_version_2_config(path)) .map_err(|e| other_io_error(format!("Error removing COL_SESSION_WINDOW_DATA {:?}", e)))?; - Ok(()) + Ok(3) +} + +/// Remove the lock file. If file is locked, it will wait up to 1s. +#[cfg(test)] +pub fn remove_file_lock(path: &std::path::Path) { + use std::{io::ErrorKind, thread::sleep, time::Duration}; + + let mut lock_path = std::path::PathBuf::from(path); + lock_path.push("lock"); + + for _ in 0..10 { + let result = std::fs::remove_file(lock_path.as_path()); + match result { + Err(error) => match error.kind() { + ErrorKind::WouldBlock => { + sleep(Duration::from_millis(100)); + continue + }, + _ => return, + }, + Ok(_) => {}, + } + } + + unreachable!("Database is locked, waited 1s for lock file: {:?}", lock_path); } #[cfg(test)] mod tests { use super::{ - columns::{v2::COL_SESSION_WINDOW_DATA, v3::*}, + columns::{v2::COL_SESSION_WINDOW_DATA, v4::*}, *, }; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; + + #[test] + fn test_paritydb_migrate_0_to_1() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + + db.commit(vec![( + COL_AVAILABILITY_META as u8, + b"5678".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 1).unwrap(); + + let db = Db::open(&paritydb_version_1_config(&path)).unwrap(); + assert_eq!( + db.get(COL_AVAILABILITY_META as u8, b"5678").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + } + + #[test] + fn test_paritydb_migrate_1_to_2() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "1").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_1_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_DISPUTE_COORDINATOR_DATA as u8, + b"1234".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v1::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 2).unwrap(); + + let db = Db::open(&paritydb_version_2_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + + assert_eq!( + db.get(COL_DISPUTE_COORDINATOR_DATA as u8, b"1234").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + + // Test we can write the new column. + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + b"1337".to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + // Read back data from new column. + assert_eq!( + db.get(COL_SESSION_WINDOW_DATA as u8, b"1337").unwrap(), + Some("0xdeadb00b".as_bytes().to_vec()) + ); + } #[test] fn test_rocksdb_migrate_1_to_2() { @@ -338,7 +536,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "1").expect("Failed to write DB version"); { - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); db.write(DBTransaction { ops: vec![DBOp::Insert { col: COL_DISPUTE_COORDINATOR_DATA, @@ -349,14 +547,14 @@ mod tests { .unwrap(); } - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 2).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS); - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); assert_eq!( db.get(COL_DISPUTE_COORDINATOR_DATA, b"1234").unwrap(), @@ -380,6 +578,108 @@ mod tests { ); } + #[test] + fn test_migrate_3_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_sanity_check; + use polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + let db_cfg: DatabaseConfig = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + + let approval_cfg = ApprovalDbConfig { + col_approval_data: crate::parachains_db::REAL_COLUMNS.col_approval_data, + }; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); + let expected_candidates = { + let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); + let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + // Fill the approval voting column with test data. + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg).unwrap() + }; + + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); + + v1_to_v2_sanity_check(std::sync::Arc::new(db), approval_cfg, expected_candidates).unwrap(); + } + + #[test] + fn test_rocksdb_migrate_0_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + + fs::write(version_file_path(db_dir.path()), "0").expect("Failed to write DB version"); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS); + } + + #[test] + fn test_paritydb_migrate_0_to_4() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "0").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v0::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 4).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS as u8); + } + + #[test] + fn test_paritydb_migrate_2_to_3() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + let test_key = b"1337"; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "2").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_2_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + test_key.to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 3).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v3::NUM_COLUMNS as u8); + } + #[test] fn test_rocksdb_migrate_2_to_3() { use kvdb_rocksdb::{Database, DatabaseConfig}; @@ -387,6 +687,7 @@ mod tests { let db_dir = tempfile::tempdir().unwrap(); let db_path = db_dir.path().to_str().unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); + { let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS as u32); @@ -395,7 +696,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "2").expect("Failed to write DB version"); - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 3).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); diff --git a/node/service/src/tests.rs b/node/service/src/tests.rs index 86119662d9bc..26c8083185d8 100644 --- a/node/service/src/tests.rs +++ b/node/service/src/tests.rs @@ -17,7 +17,7 @@ use super::{relay_chain_selection::*, *}; use futures::channel::oneshot::Receiver; -use polkadot_node_primitives::approval::VrfSignature; +use polkadot_node_primitives::approval::v2::VrfSignature; use polkadot_node_subsystem::messages::{AllMessages, BlockDescription}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml index 1fb9ac83b780..7c5546741dff 100644 --- a/node/subsystem-types/Cargo.toml +++ b/node/subsystem-types/Cargo.toml @@ -24,3 +24,4 @@ smallvec = "1.8.0" substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" } thiserror = "1.0.31" async-trait = "0.1.57" +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index 8adc39eed56d..f561a0e28512 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -32,7 +32,10 @@ use polkadot_node_network_protocol::{ self as net_protocol, peer_set::PeerSet, request_response::Requests, PeerId, }; use polkadot_node_primitives::{ - approval::{BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::{BlockApprovalMeta, IndirectSignedApprovalVote}, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, SignedDisputeStatement, SignedFullStatement, SignedFullStatementWithPVD, SubmitCollationParams, @@ -833,6 +836,8 @@ pub enum AssignmentCheckError { InvalidCert(ValidatorIndex, String), #[error("Internal state mismatch: {0:?}, {1:?}")] Internal(Hash, CandidateHash), + #[error("Oversized candidate or core bitfield >= {0}")] + InvalidBitfield(usize), } /// The result type of [`ApprovalVotingMessage::CheckAndImportApproval`] request. @@ -898,8 +903,8 @@ pub enum ApprovalVotingMessage { /// Check if the assignment is valid and can be accepted by our view of the protocol. /// Should not be sent unless the block hash is known. CheckAndImportAssignment( - IndirectAssignmentCert, - CandidateIndex, + IndirectAssignmentCertV2, + CandidateBitfield, oneshot::Sender, ), /// Check if the approval vote is valid and can be accepted by our view of the @@ -934,7 +939,7 @@ pub enum ApprovalDistributionMessage { NewBlocks(Vec), /// Distribute an assignment cert from the local validator. The cert is assumed /// to be valid, relevant, and for the given relay-parent and validator index. - DistributeAssignment(IndirectAssignmentCert, CandidateIndex), + DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. diff --git a/node/subsystem-util/src/reputation.rs b/node/subsystem-util/src/reputation.rs index 89e3eb64df9b..35746dd5fef3 100644 --- a/node/subsystem-util/src/reputation.rs +++ b/node/subsystem-util/src/reputation.rs @@ -25,6 +25,7 @@ use std::{collections::HashMap, time::Duration}; /// Default delay for sending reputation changes pub const REPUTATION_CHANGE_INTERVAL: Duration = Duration::from_secs(30); +const LOG_TARGET: &'static str = "parachain::reputation-aggregator"; type BatchReputationChange = HashMap; @@ -75,6 +76,10 @@ impl ReputationAggregator { peer_id: PeerId, rep: UnifiedReputationChange, ) { + if rep.cost_or_benefit() < 0 { + gum::debug!(target: LOG_TARGET, peer = ?peer_id, ?rep, "Modify reputation"); + } + if (self.send_immediately_if)(rep) { self.single_send(sender, peer_id, rep).await; } else { diff --git a/roadmap/implementers-guide/src/node/approval/approval-voting.md b/roadmap/implementers-guide/src/node/approval/approval-voting.md index 88744e50cf79..375b8f1f12b3 100644 --- a/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -49,22 +49,27 @@ struct TrancheEntry { assignments: Vec<(ValidatorIndex, Tick)>, } -struct OurAssignment { - cert: AssignmentCert, - tranche: DelayTranche, - validator_index: ValidatorIndex, - triggered: bool, +pub struct OurAssignment { + /// Our assignment certificate. + cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + triggered: bool, } -struct ApprovalEntry { - tranches: Vec, // sorted ascending by tranche number. - backing_group: GroupIndex, - our_assignment: Option, - our_approval_sig: Option, - assignments: Bitfield, // n_validators bits - approved: bool, +pub struct ApprovalEntry { + tranches: Vec, // sorted ascending by tranche number. + backing_group: GroupIndex, + our_assignment: Option, + our_approval_sig: Option, + assigned_validators: Bitfield, // `n_validators` bits. + approved: bool, } + struct CandidateEntry { candidate: CandidateReceipt, session: SessionIndex, @@ -200,6 +205,8 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. + * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. diff --git a/roadmap/implementers-guide/src/protocol-approval.md b/roadmap/implementers-guide/src/protocol-approval.md index 693822ce0797..aa513c16292d 100644 --- a/roadmap/implementers-guide/src/protocol-approval.md +++ b/roadmap/implementers-guide/src/protocol-approval.md @@ -98,12 +98,14 @@ We want checkers for candidate equivocations that lie outside our preferred rela Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both `RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: `RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the sampled availability core in this relay chain block. We choose three samples initially, but we could make polkadot more secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. + There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. `RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has `zeroth_delay_tranche_width+1` times as many assignments as any other tranche. @@ -114,9 +116,9 @@ As future work (or TODO?), we should merge assignment notices with the same dela We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` assignments. +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing their assignments. @@ -164,7 +166,7 @@ We need the chain to win in this case, but doing this requires imposing an annoy ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been properly analyzed. +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per tranche and the zeroth delay tranche width. diff --git a/roadmap/implementers-guide/src/types/approval.md b/roadmap/implementers-guide/src/types/approval.md index b58e0a8187e1..2cc9b34cc700 100644 --- a/roadmap/implementers-guide/src/types/approval.md +++ b/roadmap/implementers-guide/src/types/approval.md @@ -20,6 +20,35 @@ enum AssignmentCertKind { } } +enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block where the + /// candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, +} + struct AssignmentCert { // The criterion which is claimed to be met by this cert. kind: AssignmentCertKind, diff --git a/roadmap/implementers-guide/src/types/runtime.md b/roadmap/implementers-guide/src/types/runtime.md index 79da899bd35e..4b97409f8df3 100644 --- a/roadmap/implementers-guide/src/types/runtime.md +++ b/roadmap/implementers-guide/src/types/runtime.md @@ -55,7 +55,7 @@ struct HostConfiguration { pub zeroth_delay_tranche_width: u32, /// The number of validators needed to approve a block. pub needed_approvals: u32, - /// The number of samples to do of the RelayVRFModulo approval assignment criterion. + /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. pub relay_vrf_modulo_samples: u32, /// Total number of individual messages allowed in the parachain -> relay-chain message queue. pub max_upward_queue_count: u32, diff --git a/scripts/ci/gitlab/pipeline/zombienet.yml b/scripts/ci/gitlab/pipeline/zombienet.yml index 62e081d1de0e..1c8df44cbaa7 100644 --- a/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/scripts/ci/gitlab/pipeline/zombienet.yml @@ -160,6 +160,34 @@ zombienet-tests-parachains-disputes-past-session: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-max-tranche0-approvals: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + RUN_IN_CONTAINER: "1" + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0005-parachains-max-tranche0.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test zombienet-test-parachains-upgrade-smoke-test: stage: zombienet @@ -182,7 +210,7 @@ zombienet-test-parachains-upgrade-smoke-test: - echo "${GH_DIR}" - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" diff --git a/zombienet_tests/functional/0005-parachains-max-tranche0.toml b/zombienet_tests/functional/0005-parachains-max-tranche0.toml new file mode 100644 index 000000000000..ab7fa0195d13 --- /dev/null +++ b/zombienet_tests/functional/0005-parachains-max-tranche0.toml @@ -0,0 +1,40 @@ +[settings] +timeout = 1000 +bootnode = true + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + max_validators_per_core = 1 + needed_approvals = 7 + relay_vrf_modulo_samples = 5 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" +default_command = "polkadot" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.node_groups]] + name = "some-validator" + count = 8 + args = ["-lparachain=debug,runtime=debug"] + +{% for id in range(2000,2005) %} +[[parachains]] +id = {{id}} +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size={{10000*(id-1999)}} --pvf-complexity={{id - 1999}}" + [parachains.collator] + image = "{{COL_IMAGE}}" + name = "collator" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size={{10000*(id-1999)}}", "--parachain-id={{id}}", "--pvf-complexity={{id - 1999}}"] +{% endfor %} + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" diff --git a/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl b/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl new file mode 100644 index 000000000000..48a56f60c9bc --- /dev/null +++ b/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl @@ -0,0 +1,27 @@ +Description: Test if parachains make progress with most of approvals being tranch0 +Network: ./0005-parachains-max-tranche0.toml +Creds: config + +# Check authority status. +some-validator-0: reports node_roles is 4 +some-validator-1: reports node_roles is 4 +some-validator-3: reports node_roles is 4 +some-validator-4: reports node_roles is 4 +some-validator-5: reports node_roles is 4 +some-validator-6: reports node_roles is 4 +some-validator-7: reports node_roles is 4 + +some-validator-0: parachain 2000 block height is at least 5 within 180 seconds +some-validator-1: parachain 2001 block height is at least 5 within 180 seconds +some-validator-2: parachain 2002 block height is at least 5 within 180 seconds +some-validator-3: parachain 2003 block height is at least 5 within 180 seconds +some-validator-4: parachain 2004 block height is at least 5 within 180 seconds + +some-validator-0: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-1: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-2: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-3: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-4: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-5: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-6: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-7: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 diff --git a/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml b/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml index 0becb408550a..88b789f37fa1 100644 --- a/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml +++ b/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml @@ -31,7 +31,7 @@ cumulus_based = true [parachains.collator] name = "collator01" image = "{{COL_IMAGE}}" - command = "polkadot-collator" + command = "polkadot-parachain" [[parachains.collator.env]] name = "RUST_LOG"