Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

approval-voting improvement: include all tranche0 assignments in one certificate #6782

Open
wants to merge 118 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 91 commits
Commits
Show all changes
118 commits
Select commit Hold shift + click to select a range
4adfc1f
Add compact vrf modulo assignment
sandreim Feb 7, 2023
71bf07b
compute all assignments from a single vrf output
sandreim Feb 7, 2023
b1aabc9
impl compute/check of `RelayVRFModuloCompact`
sandreim Feb 7, 2023
8b881ee
fmt and missed file
sandreim Feb 7, 2023
81e62ec
check_and_import_assignment: multiple candidates
sandreim Feb 8, 2023
82c2b5f
integration step 1: subsystem message changes
sandreim Feb 10, 2023
d9d5681
Approval distribution WIP
sandreim Feb 10, 2023
596a347
approval-dist: fixed compilation errors
sandreim Feb 10, 2023
c87502b
test fixups, WIP
sandreim Feb 10, 2023
cc9f63a
itertools
sandreim Feb 13, 2023
eb86e56
Allow claimed cores to be a subset of vrf output assignments
sandreim Feb 13, 2023
4d421d2
Revert approval distribution changes
sandreim Feb 13, 2023
7b6db76
WIP
sandreim Feb 15, 2023
46d76d4
Fix compile errors
sandreim Feb 16, 2023
b18295e
add todo
sandreim Feb 16, 2023
a41254c
Keep track of peer protocol version
sandreim Feb 17, 2023
3bb8823
network protocol bump and addition
sandreim Feb 20, 2023
b34788e
Approval distribution handing of v1 vs v2
sandreim Feb 20, 2023
48109ec
Add VStaging network protocol
sandreim Feb 23, 2023
b672d86
comment aggression metrics
sandreim Feb 23, 2023
2a300de
finish impl v2/v1 sending
sandreim Feb 24, 2023
7858ad5
fix view change handling typo bug
sandreim Feb 24, 2023
0cf73fd
Rename leftover V2 to vstaging
sandreim Feb 24, 2023
412282a
Disable assignments V2
sandreim Feb 24, 2023
af3b64b
update todo, docs
sandreim Feb 24, 2023
6612b3c
fmt
sandreim Feb 24, 2023
f8ccec9
Temporarly disable CI cargo test to get a build for burn-in
sandreim Feb 24, 2023
ef42e0c
debug -> trace
sandreim Feb 24, 2023
0c31d61
enable v2 assignments
sandreim Feb 24, 2023
e3598b1
Remove unimplemented!
sandreim Mar 10, 2023
24db36c
Metric updates
sandreim Mar 10, 2023
741e91d
approval-voting: simplify v2 assignments (#6802)
sandreim Mar 31, 2023
7bf83b1
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Mar 31, 2023
01b6ec0
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Apr 3, 2023
1a38026
fix merge damage
sandreim Apr 8, 2023
b0402a3
Include core bitfield in compact assignments
sandreim Apr 18, 2023
2ad5234
Fix existing approval voting tests
sandreim Apr 20, 2023
6a8f17a
Add bitfield certificate extra check
sandreim Apr 20, 2023
a8f952c
Approval dist test compilation fixes
sandreim Apr 20, 2023
31c0548
Uncomment aggression
sandreim Apr 20, 2023
ef0bca5
approval-dist: some tests fail, but it compiles
sandreim Apr 20, 2023
1a7d33b
Re-enable tests
sandreim Apr 21, 2023
d2fd321
Fix tests
sandreim Apr 21, 2023
2288086
Fix build
sandreim Apr 21, 2023
5755b17
clippy fixes
sandreim Apr 23, 2023
fc43f5b
fix more tests :(
sandreim Apr 24, 2023
33359cb
more network protocol test fixes
sandreim Apr 25, 2023
c8af81a
Fix network bridge tests using code from async branch
sandreim Apr 25, 2023
d0d1209
Merge holy master branch :church:
sandreim Apr 25, 2023
510c28a
heal merge damage :hospital:
sandreim Apr 27, 2023
32ab33a
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Apr 27, 2023
9f6f0e6
Bump test timeout to make CI happy when slow
sandreim May 2, 2023
274ea46
approval primitives v1/v2 refactor
sandreim May 11, 2023
9f66ed1
remove leftover sleep in CI test pipeline
sandreim May 11, 2023
6274c37
Fix tests and enable v2 assignments in tests
sandreim May 11, 2023
28dd203
add/modify approval distribution tests for v2
sandreim May 17, 2023
5c5b86b
Add another approval distribution test
sandreim May 17, 2023
7f51489
add zombienet test
sandreim May 17, 2023
f4a983b
fix tests build
sandreim May 17, 2023
180273a
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 18, 2023
2a65a98
fix build
sandreim May 18, 2023
88b4349
enable v2 assignemnts for testing
sandreim May 18, 2023
424ea57
Add back removed comment
sandreim May 18, 2023
229d1c1
review feedback 1/2
sandreim May 22, 2023
9152a9c
Full approval db migration to v2
sandreim May 24, 2023
0f9e1ec
fix comment
sandreim May 24, 2023
fe5b548
Fix AcceptedDuplicate, test and import log
sandreim May 25, 2023
a4fae3f
clippy
sandreim May 25, 2023
2be89ff
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 25, 2023
14c3643
fmt
sandreim May 25, 2023
ad3875d
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim May 25, 2023
0bc5aa8
Fix `get_approval_signatures`
sandreim May 26, 2023
574c92b
Add ApprovalEntry::get_approval
sandreim May 26, 2023
4c04699
disable v2 assignments
sandreim May 26, 2023
0be0201
review feedback and print remove
sandreim May 26, 2023
496f0c5
Fix logging
sandreim May 31, 2023
ba6078c
fix
sandreim May 31, 2023
bfce579
Get rid of old "legacy" protocols, not used anymore
sandreim Jun 8, 2023
d012e7b
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Jun 8, 2023
cd911e6
merge fixes
sandreim Jun 8, 2023
959e529
Refactor and always send on main protocol version
sandreim Jun 9, 2023
a8e3f28
Fix db upgrade after merge and fmt
sandreim Jun 9, 2023
78de04e
Fix zombienet merge damage
sandreim Jun 9, 2023
112b0be
Implement seq updates 0->4 now possible for parachainsDB
sandreim Jun 12, 2023
4b1743c
add/fix db upgrade tests
sandreim Jun 13, 2023
36f6b39
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Jun 14, 2023
4e0fac6
update colander image version
pepoviola Jun 14, 2023
7f97355
update image to polkadot-parachain
pepoviola Jun 14, 2023
d356081
review
sandreim Jul 4, 2023
a179646
fmt
sandreim Jul 4, 2023
de6ea33
Remove superfuous `assignment_bitfield` from `OurAssignment`
sandreim Jul 4, 2023
21e53ee
feedback
sandreim Jul 24, 2023
fae7e52
remove old comment
sandreim Jul 24, 2023
b5cf4b8
fix build
sandreim Jul 27, 2023
6e25ae2
Fix get_assignment_core_indices()
sandreim Jul 27, 2023
5b565ed
some TODOs done
sandreim Jul 27, 2023
59a8a59
Merge branch 'sandreim/vrf_modulo_comapct_assignment' of github.com:p…
sandreim Jul 27, 2023
8436ada
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Jul 27, 2023
910e1ec
fix test build
sandreim Jul 27, 2023
58e028a
review feedback
sandreim Jul 28, 2023
102b307
use V1ReadBackend
sandreim Jul 28, 2023
a8eae59
sanitize bitfields
sandreim Jul 31, 2023
8420b90
remove print
sandreim Jul 31, 2023
ecad9ec
Add network-staging feature
sandreim Aug 1, 2023
f61d8ab
fix so DistributeAssignment gets sent once for compact assignments
sandreim Aug 2, 2023
6eefe3f
review feedback + fixes
sandreim Aug 8, 2023
a88dc80
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Aug 8, 2023
509125f
fix test build
sandreim Aug 9, 2023
bfab907
review feedback
sandreim Aug 21, 2023
1964263
Merge branch 'master' of github.com:paritytech/polkadot into sandreim…
sandreim Aug 21, 2023
6090974
fix Cargo.lock
sandreim Aug 21, 2023
4841044
remove unreachable
sandreim Aug 21, 2023
c9c7d3c
Fix zombienet test
sandreim Aug 21, 2023
a9228c0
happy clippy
sandreim Aug 22, 2023
d7dffc9
Fix flaky paritydb upgrade tests - wait lock file
sandreim Aug 22, 2023
e81cc9f
remove unused
sandreim Aug 22, 2023
4619a2b
Extra logging
sandreim Aug 22, 2023
8a4a2db
fix approval import and add more rep logs
sandreim Aug 23, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions node/core/approval-voting/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ schnorrkel = "0.9.1"
kvdb = "0.13.0"
derive_more = "0.99.17"
thiserror = "1.0.31"
itertools = "0.10.5"

polkadot-node-subsystem = { path = "../../subsystem" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
Expand All @@ -29,6 +30,9 @@ sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "mast
sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = ["full_crypto"] }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
# Needed for migration test helpers
test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
rand_core = "0.5.1"

[dev-dependencies]
async-trait = "0.1.57"
Expand Down
60 changes: 30 additions & 30 deletions node/core/approval-voting/src/approval_checking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
//! Utilities for checking whether a candidate has been approved under a given block.

use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice};
use polkadot_node_primitives::approval::DelayTranche;
use polkadot_node_primitives::approval::v1::DelayTranche;
use polkadot_primitives::ValidatorIndex;

use crate::{
Expand Down Expand Up @@ -472,9 +472,9 @@ mod tests {
}
.into();

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: BitVec::default(),
assigned_validators: BitVec::default(),
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -509,22 +509,22 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 0,
assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 2,
assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -581,22 +581,22 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 0,
assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 2,
assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -647,9 +647,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 5],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -691,9 +691,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -731,9 +731,9 @@ mod tests {
let no_show_duration = 10;
let needed_approvals = 4;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; 10],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -776,9 +776,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -843,9 +843,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -934,9 +934,9 @@ mod tests {
let needed_approvals = 4;
let n_validators = 8;

let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -1041,15 +1041,15 @@ mod tests {
candidate.mark_approval(ValidatorIndex(i));
}

let approval_entry = approval_db::v1::ApprovalEntry {
let approval_entry = approval_db::v2::ApprovalEntry {
tranches: vec![
// Assignments with invalid validator indexes.
approval_db::v1::TrancheEntry {
approval_db::v2::TrancheEntry {
tranche: 1,
assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
},
],
assignments: bitvec![u8, BitOrderLsb0; 1; 3],
assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3],
our_assignment: None,
our_approval_sig: None,
backing_group: GroupIndex(0),
Expand Down Expand Up @@ -1094,12 +1094,12 @@ mod tests {
];

for test_tranche in test_tranches {
let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
tranches: Vec::new(),
backing_group: GroupIndex(0),
our_assignment: None,
our_approval_sig: None,
assignments: bitvec![u8, BitOrderLsb0; 0; 3],
assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3],
approved: false,
}
.into();
Expand Down
1 change: 1 addition & 0 deletions node/core/approval-voting/src/approval_db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@
//! time being we share the same DB with the rest of Substrate.

pub mod v1;
pub mod v2;
Loading