From ddfac4d9b64506fed6a038dc777b80e0d11ac21f Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 23 Apr 2024 15:54:35 -0300 Subject: [PATCH 1/3] perf: use Arc<[P]> for Polynomials --- Cargo.toml | 1 + rust-toolchain.toml | 2 +- src/cs/implementations/copy_permutation.rs | 48 ++++--- src/cs/implementations/fast_serialization.rs | 45 +++++++ src/cs/implementations/lookup_argument.rs | 7 +- .../implementations/lookup_argument_in_ext.rs | 44 +++--- src/cs/implementations/polynomial/lde.rs | 43 +++--- src/cs/implementations/polynomial/mod.rs | 70 ++++++---- src/cs/implementations/prover.rs | 84 +++++------- src/cs/implementations/setup.rs | 17 ++- src/cs/implementations/utils.rs | 33 +++-- src/cs/implementations/witness.rs | 25 ++-- src/lib.rs | 2 + src/utils.rs | 125 ++++++++++++++++++ 14 files changed, 372 insertions(+), 174 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3d658ff..d9cadaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ lazy_static = "*" arrayvec = "0.7" const_format = "0.2" bincode = "*" +ecow = "*" ethereum-types = "=0.14.1" cs_derive = { path = "./cs_derive" } itertools = "0.10" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 6b48c00..c2bbf59 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-06-25" +channel = "nightly-2023-08-01" diff --git a/src/cs/implementations/copy_permutation.rs b/src/cs/implementations/copy_permutation.rs index a0a041c..20a8f8b 100644 --- a/src/cs/implementations/copy_permutation.rs +++ b/src/cs/implementations/copy_permutation.rs @@ -11,6 +11,8 @@ use crate::{ }; use crate::{cs::traits::GoodAllocator, field::PrimeField}; +use std::sync::Arc; + pub fn num_intermediate_partial_product_relations( num_copys_under_copy_permutation: usize, quotient_degree: usize, @@ -60,10 +62,11 @@ pub(crate) fn pointwise_rational< { let non_residue = P::constant(*non_residue, ctx); worker.scope(typical_size, |scope, chunk_size| { + let basis_storage = unsafe { Arc::get_mut_unchecked(&mut basis.storage) }; for (((w, sigma), x_poly), dst) in (witness_poly.storage.chunks(chunk_size)) .zip(sigma_poly.storage.chunks(chunk_size)) .zip(precomputed_x_poly.storage.chunks(chunk_size)) - .zip(basis.storage.chunks_mut(chunk_size)) + .zip(basis_storage.chunks_mut(chunk_size)) { let mut ctx = *ctx; scope.spawn(move |_| { @@ -149,12 +152,16 @@ pub(crate) fn pointwise_rational_in_extension< { let non_residue = P::constant(*non_residue, ctx); worker.scope(typical_size, |scope, chunk_size| { + let (basis_c0_storage, basis_c1_storage) = unsafe {( + Arc::get_mut_unchecked(&mut basis_c0.storage), + Arc::get_mut_unchecked(&mut basis_c1.storage), + )}; for ((((w, sigma), x_poly), dst_c0), dst_c1) in (witness_poly.storage.chunks(chunk_size)) .zip(sigma_poly.storage.chunks(chunk_size)) .zip(precomputed_x_poly.storage.chunks(chunk_size)) - .zip(basis_c0.storage.chunks_mut(chunk_size)) - .zip(basis_c1.storage.chunks_mut(chunk_size)) + .zip(basis_c0_storage.chunks_mut(chunk_size)) + .zip(basis_c1_storage.chunks_mut(chunk_size)) { let mut ctx = *ctx; scope.spawn(move |_| { @@ -281,8 +288,9 @@ pub(crate) fn pointwise_product_into< for source in inputs.iter() { worker.scope(typical_size, |scope, chunk_size| { + let into_storage = unsafe { Arc::get_mut_unchecked(&mut into.storage)}; for (dst, src) in - (into.storage.chunks_mut(chunk_size)).zip(source.storage.chunks(chunk_size)) + (into_storage.chunks_mut(chunk_size)).zip(source.storage.chunks(chunk_size)) { let mut ctx = *ctx; scope.spawn(move |_| { @@ -336,13 +344,23 @@ pub(crate) fn pointwise_product_in_extension_into< ) { let typical_size = into_c0.storage.len(); // we need raw length in counts of P + let (into_c0_storage, into_c1_storage) = unsafe {( + Arc::get_mut_unchecked(&mut into_c0.storage), + Arc::get_mut_unchecked(&mut into_c1.storage), + )}; + + // 18: 0x5617ec14977f - boojum::cs::implementations::copy_permutation::pointwise_product_in_extension_into::h7afc923f8b42188e + // 19: 0x5617ec14a7b6 - boojum::cs::implementations::copy_permutation::compute_partial_products_in_extension::h2e6bd4b1c53ed759 + // 20: 0x5617ebcb9319 - boojum::cs::implementations::prover::>::prove_cpu_basic::h088688d6cd7fed3a + // 21: 0x5617ebbdff22 - boojum::cs::implementations::convenience::>::prove_from_precomputations::hb5faa7c379034a0a + // 22: 0x5617ebeecd3a - zkevm_test_harness::prover_utils::prove_base_layer_circuit::h6ca12e453ce37157 + for source in inputs.iter() { let [src_c0, src_c1] = source; worker.scope(typical_size, |scope, chunk_size| { - for (((dst_c0, dst_c1), src_c0), src_c1) in into_c0 - .storage + for (((dst_c0, dst_c1), src_c0), src_c1) in into_c0_storage .chunks_mut(chunk_size) - .zip(into_c1.storage.chunks_mut(chunk_size)) + .zip(into_c1_storage.chunks_mut(chunk_size)) .zip(src_c0.storage.chunks(chunk_size)) .zip(src_c1.storage.chunks(chunk_size)) { @@ -625,8 +643,7 @@ pub(crate) fn compute_partial_products< // we have to apply pointwise products on top of Z(x) - for el in partial_elementwise_products.into_iter() { - let mut el = el; + for mut el in partial_elementwise_products.into_iter() { pointwise_product_into(&previous, &mut el, worker, ctx); // we have new pointwise in el, and untouched previous, so we can reuse the storage @@ -756,8 +773,7 @@ pub(crate) fn compute_partial_products_in_extension< // we have to apply pointwise products on top of Z(x) - for el in partial_elementwise_products.into_iter() { - let [mut el_c0, mut el_c1] = el; + for [mut el_c0, mut el_c1] in partial_elementwise_products.into_iter() { pointwise_product_in_extension_into::( &previous, &mut el_c0, &mut el_c1, worker, ctx, ); @@ -1232,12 +1248,14 @@ pub(crate) fn compute_quotient_terms_in_extension< } } - unsafe { std::sync::Arc::get_mut_unchecked(&mut dst_c0.storage[outer]) } - .storage[inner] + unsafe { + Arc::get_mut_unchecked(&mut dst_c0.storage[outer].storage)[inner] + } .add_assign(&contribution_c0, &mut ctx); - unsafe { std::sync::Arc::get_mut_unchecked(&mut dst_c1.storage[outer]) } - .storage[inner] + unsafe { + Arc::get_mut_unchecked(&mut dst_c1.storage[outer].storage)[inner] + } .add_assign(&contribution_c1, &mut ctx); iterator.advance(); diff --git a/src/cs/implementations/fast_serialization.rs b/src/cs/implementations/fast_serialization.rs index ea7301c..85bbc77 100644 --- a/src/cs/implementations/fast_serialization.rs +++ b/src/cs/implementations/fast_serialization.rs @@ -1,4 +1,5 @@ use core::slice; +use ecow::EcoVec; use std::{ alloc::Allocator, error::Error, @@ -135,6 +136,50 @@ impl MemcopySerializable for std::sync::Arc { } } +// FIXME: write a more optimized version :) +impl< + F: SmallField, + P: crate::field::traits::field_like::PrimeFieldLikeVectorized, + > MemcopySerializable for EcoVec

+{ + fn write_into_buffer(&self, dst: W) -> Result<(), Box> { + let mut as_vec = Vec::with_capacity(self.len()); + as_vec.extend_from_slice(self.as_slice()); + MemcopySerializable::write_into_buffer(&as_vec, dst)?; + + Ok(()) + } + + fn read_from_buffer(src: R) -> Result> { + let vec: Vec

= MemcopySerializable::read_from_buffer(src)?; + Ok(vec.as_slice().into()) + } +} + +impl< + F: SmallField, + P: crate::field::traits::field_like::PrimeFieldLikeVectorized, + A: GoodAllocator + 'static, + > MemcopySerializable for std::sync::Arc<[P], A> +where Vec: MemcopySerializable +{ + fn write_into_buffer(&self, dst: W) -> Result<(), Box> { + MemcopySerializable::write_into_buffer(&self.as_ref().to_owned(), dst)?; + + Ok(()) + } + + fn read_from_buffer(src: R) -> Result> { + let vec: Vec = MemcopySerializable::read_from_buffer(src)?; + let arc_slice = unsafe { + let mut arc_slice = std::sync::Arc::new_uninit_slice_in(vec.len(), A::default()); + std::mem::MaybeUninit::write_slice(std::sync::Arc::get_mut_unchecked(&mut arc_slice), vec.as_slice()); + arc_slice.assume_init() + }; + Ok(arc_slice) + } +} + // Prime field like vectors are the special case, and it's only implemented for vector! impl< diff --git a/src/cs/implementations/lookup_argument.rs b/src/cs/implementations/lookup_argument.rs index 90fdaa3..9f9ee53 100644 --- a/src/cs/implementations/lookup_argument.rs +++ b/src/cs/implementations/lookup_argument.rs @@ -67,7 +67,8 @@ pub(crate) fn compute_lookup_poly_pairs_over_general_purpose_columns< let mut selector: GenericPolynomial = (*constant_polys[0]).clone(); if lookup_selector_path[0] == false { worker.scope(selector.storage.len(), |scope, chunk_size| { - for dst in selector.storage.chunks_mut(chunk_size) { + let selector_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut selector.storage) }; + for dst in selector_storage.chunks_mut(chunk_size) { let mut ctx = *ctx; scope.spawn(move |_| { // inverse @@ -87,8 +88,8 @@ pub(crate) fn compute_lookup_poly_pairs_over_general_purpose_columns< .zip(constant_polys[1..].iter()) { worker.scope(selector.storage.len(), |scope, chunk_size| { - for (dst, src) in selector - .storage + let selector_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut selector.storage) }; + for (dst, src) in selector_storage .chunks_mut(chunk_size) .zip(src.storage.chunks(chunk_size)) { diff --git a/src/cs/implementations/lookup_argument_in_ext.rs b/src/cs/implementations/lookup_argument_in_ext.rs index decfdf5..fe876fa 100644 --- a/src/cs/implementations/lookup_argument_in_ext.rs +++ b/src/cs/implementations/lookup_argument_in_ext.rs @@ -1082,20 +1082,16 @@ pub(crate) fn compute_quotient_terms_for_lookup_specialized< // so we just add unsafe { - std::sync::Arc::get_mut_unchecked( - &mut aggregated_lookup_columns_c0.storage[outer], - ) - .storage[inner] - .add_assign(&tmp_c0, &mut ctx); - }; + std::sync::Arc::get_mut_unchecked(&mut aggregated_lookup_columns_c0.storage[outer].storage) + [inner] + } + .add_assign(&tmp_c0, &mut ctx); unsafe { - std::sync::Arc::get_mut_unchecked( - &mut aggregated_lookup_columns_c1.storage[outer], - ) - .storage[inner] - .add_assign(&tmp_c1, &mut ctx); - }; + std::sync::Arc::get_mut_unchecked(&mut aggregated_lookup_columns_c1.storage[outer].storage) + [inner] + } + .add_assign(&tmp_c1, &mut ctx); lde_iter.advance(); } @@ -1211,15 +1207,15 @@ pub(crate) fn compute_quotient_terms_for_lookup_specialized< // add into accumulator unsafe { - std::sync::Arc::get_mut_unchecked(&mut dst_c0.storage[outer]).storage + std::sync::Arc::get_mut_unchecked(&mut dst_c0.storage[outer].storage) [inner] - .add_assign(&tmp_c0, &mut ctx); - }; + } + .add_assign(&tmp_c0, &mut ctx); unsafe { - std::sync::Arc::get_mut_unchecked(&mut dst_c1.storage[outer]).storage + std::sync::Arc::get_mut_unchecked(&mut dst_c1.storage[outer].storage) [inner] - .add_assign(&tmp_c1, &mut ctx); - }; + } + .add_assign(&tmp_c1, &mut ctx); lde_iter.advance(); } @@ -1300,15 +1296,15 @@ pub(crate) fn compute_quotient_terms_for_lookup_specialized< // add into accumulator unsafe { - std::sync::Arc::get_mut_unchecked(&mut dst_c0.storage[outer]).storage + std::sync::Arc::get_mut_unchecked(&mut dst_c0.storage[outer].storage) [inner] - .add_assign(&tmp_c0, &mut ctx); - }; + } + .add_assign(&tmp_c0, &mut ctx); unsafe { - std::sync::Arc::get_mut_unchecked(&mut dst_c1.storage[outer]).storage + std::sync::Arc::get_mut_unchecked(&mut dst_c1.storage[outer].storage) [inner] - .add_assign(&tmp_c1, &mut ctx); - }; + } + .add_assign(&tmp_c1, &mut ctx); lde_iter.advance(); } diff --git a/src/cs/implementations/polynomial/lde.rs b/src/cs/implementations/polynomial/lde.rs index c8fcbe0..6878c1d 100644 --- a/src/cs/implementations/polynomial/lde.rs +++ b/src/cs/implementations/polynomial/lde.rs @@ -1,7 +1,6 @@ use crate::cs::traits::GoodAllocator; use super::*; -use std::sync::Arc; #[derive(Derivative)] #[derivative(Clone, Debug, PartialEq, Eq)] @@ -164,9 +163,9 @@ pub struct ArcGenericLdeStorage< A: GoodAllocator = Global, B: GoodAllocator = Global, > { - #[serde(serialize_with = "crate::utils::serialize_vec_arc")] - #[serde(deserialize_with = "crate::utils::deserialize_vec_arc")] - pub storage: Vec>, B>, + #[serde(serialize_with = "crate::utils::serialize_vec_with_allocator")] + #[serde(deserialize_with = "crate::utils::deserialize_vec_with_allocator")] + pub storage: Vec, B>, } pub type ArcLdeStorage = ArcGenericLdeStorage; @@ -207,7 +206,7 @@ where for _ in 0..capacity { let inner: GenericPolynomial = MemcopySerializable::read_from_buffer(&mut src)?; - storage.push(Arc::new(inner)); + storage.push(inner); } let new = Self { storage }; @@ -245,7 +244,7 @@ impl< let mut inner = Vec::with_capacity_in(inner_size, inner_allocator.clone()); inner.resize(inner_size, P::zero(&mut ())); let as_poly = GenericPolynomial::from_storage(inner); - storage.push(Arc::new(as_poly)); + storage.push(as_poly); } Self { storage } @@ -261,12 +260,11 @@ impl< debug_assert!(inner_size.is_power_of_two()); debug_assert!(outer_size.is_power_of_two()); let mut storage = Vec::with_capacity_in(outer_size, outer_allocator); + let mut tmpl = Vec::with_capacity_in(inner_size, inner_allocator.clone()); + tmpl.resize(inner_size, P::zero(&mut ())); for _ in 0..outer_size { - let as_poly = GenericPolynomial::from_storage(Vec::with_capacity_in( - inner_size, - inner_allocator.clone(), - )); - storage.push(Arc::new(as_poly)); + let as_poly = GenericPolynomial::from_storage(tmpl.clone()); + storage.push(as_poly); } Self { storage } @@ -279,18 +277,14 @@ impl< pub unsafe fn assume_init(&mut self, inner_size: usize) { debug_assert!(inner_size.is_power_of_two()); for el in self.storage.iter_mut() { - Arc::get_mut(el).unwrap().storage.set_len(inner_size); + // FIXME + //el.storage.set_len(inner_size); } } #[inline] pub fn from_owned(owned: GenericLdeStorage) -> Self { - let mut storage = Vec::with_capacity_in(owned.storage.len(), B::default()); - for el in owned.storage.into_iter() { - storage.push(Arc::new(el)); - } - - Self { storage } + Self { storage: owned.storage } } // shallow clone, for readonly @@ -300,10 +294,7 @@ impl< assert!((self.storage.len() / degree).is_power_of_two()); let mut storage = Vec::with_capacity_in(degree, B::default()); - for i in 0..degree { - storage.push(Arc::clone(&self.storage[i])); - } - + storage.extend_from_slice(&self.storage[..degree]); Self { storage } } @@ -314,9 +305,9 @@ impl< let mut storage = Vec::with_capacity_in(degree, B::default()); for i in 0..degree { - let owned_chunk = Clone::clone(self.storage[i].as_ref()); + let owned_chunk = self.storage[i].to_owned(); debug_assert!(owned_chunk.storage.as_ptr().addr() % std::mem::align_of::

() == 0); - storage.push(Arc::new(owned_chunk)); + storage.push(owned_chunk); } Self { storage } @@ -371,9 +362,7 @@ impl< new_params.ldes_generator = new_params.ldes_generator.pow_u64(extra_pow as u64); let mut new_storage = Vec::with_capacity_in(degree, B::default()); - for i in 0..degree { - new_storage.push(Arc::clone(&self.storage.storage[i])); - } + new_storage.extend_from_slice(&self.storage.storage[..degree]); let storage = ArcGenericLdeStorage { storage: new_storage, diff --git a/src/cs/implementations/polynomial/mod.rs b/src/cs/implementations/polynomial/mod.rs index 5659da2..a32debd 100644 --- a/src/cs/implementations/polynomial/mod.rs +++ b/src/cs/implementations/polynomial/mod.rs @@ -3,6 +3,7 @@ use crate::field::traits::field_like::PrimeFieldLikeVectorized; use crate::field::PrimeField; use crate::utils::*; use std::alloc::Global; +use std::sync::Arc; use super::fast_serialization::MemcopySerializable; use super::*; @@ -42,9 +43,9 @@ pub struct GenericPolynomial< P: field::traits::field_like::PrimeFieldLikeVectorized, A: GoodAllocator = Global, > { - #[serde(serialize_with = "crate::utils::serialize_vec_with_allocator")] - #[serde(deserialize_with = "crate::utils::deserialize_vec_with_allocator")] - pub storage: Vec, + #[serde(serialize_with = "crate::utils::serialize_arc_slice")] + #[serde(deserialize_with = "crate::utils::deserialize_arc_slice")] + pub storage: Arc<[P], A>, pub _marker: std::marker::PhantomData<(F, P, FORM)>, } @@ -55,9 +56,15 @@ impl< A: GoodAllocator, > Clone for GenericPolynomial { + // NOTE: we only use the `Arc` semantics for the `ArcLdeStorage`, + // so *this* clone should be deep. #[inline(always)] fn clone(&self) -> Self { - let storage = Vec::clone(&self.storage); + let mut storage = Arc::<[P], A>::new_uninit_slice_in(self.storage.len(), A::default()); + let storage = unsafe { + std::mem::MaybeUninit::write_slice(Arc::get_mut_unchecked(&mut storage), &self.storage); + storage.assume_init() + }; Self { storage, @@ -65,9 +72,11 @@ impl< } } + // NOTE: we only use the `Arc` semantics for the `ArcLdeStorage`, + // so *this* clone should be deep. #[inline(always)] fn clone_from(&mut self, source: &Self) { - Vec::clone_from(&mut self.storage, &source.storage); + *self = source.clone(); } } @@ -125,7 +134,7 @@ impl< P: field::traits::field_like::PrimeFieldLikeVectorized, A: GoodAllocator, B: GoodAllocator, - > MemcopySerializable for Vec>, B> + > MemcopySerializable for Vec, B> where Self: 'static, { @@ -156,7 +165,7 @@ where for _ in 0..capacity { let inner: GenericPolynomial = MemcopySerializable::read_from_buffer(&mut src)?; - result.push(std::sync::Arc::new(inner)); + result.push(inner); } Ok(result) @@ -172,14 +181,6 @@ impl< A: GoodAllocator, > GenericPolynomial { - #[inline] - pub fn new() -> Self { - Self { - storage: Vec::new_in(A::default()), - _marker: std::marker::PhantomData, - } - } - pub fn pretty_compare(&self, other: &Self) { for (idx, (a, b)) in P::slice_into_base_slice(&self.storage) .iter() @@ -195,6 +196,14 @@ impl< #[inline] pub(crate) fn from_storage(storage: Vec) -> Self { debug_assert!(storage.as_ptr().addr() % std::mem::align_of::

() == 0); + let mut new_storage = Arc::<[P], A>::new_uninit_slice_in(storage.len(), A::default()); + let storage = unsafe { + let slice = Arc::get_mut_unchecked(&mut new_storage); + for (i, v) in storage.iter().enumerate() { + slice[i].as_mut_ptr().write(*v); + } + new_storage.assume_init() + }; Self { storage, _marker: std::marker::PhantomData, @@ -203,7 +212,9 @@ impl< #[inline] pub(crate) fn into_storage(self) -> Vec { - self.storage + let mut storage = Vec::with_capacity_in(self.storage.len(), A::default()); + storage.extend_from_slice(&self.storage); + storage } #[inline] @@ -222,15 +233,15 @@ impl< #[inline] pub(crate) fn clone_respecting_allignment(&self) -> Self { - let buffer = clone_respecting_allignment::<_, U, _>(&self.storage); + // FIXME: allignment Self { - storage: buffer, + storage: self.storage.to_owned(), _marker: std::marker::PhantomData, } } pub(crate) fn chunks( - self: &std::sync::Arc, + self: &Self, chunk_size: usize, ) -> Vec, B> { let len = self.storage.len(); @@ -244,7 +255,7 @@ impl< } let chunk = GenericPolynomialChunk { - over: std::sync::Arc::clone(self), + over: self.clone(), range: start..end, _marker: std::marker::PhantomData, }; @@ -270,15 +281,16 @@ impl< let num_chunks = self.domain_size() / degree; - let mut storage = self.storage; - // there is no easy way to split the allocation let mut result = Vec::with_capacity_in(num_chunks, B::default()); - for _ in 0..num_chunks { - let mut subchunk = Vec::with_capacity_in(degree / P::SIZE_FACTOR, A::default()); - subchunk.extend(storage.drain(..degree / P::SIZE_FACTOR)); + for subchunk in self.storage.chunks(degree / P::SIZE_FACTOR) { + let storage = unsafe { + let mut slice = Arc::new_uninit_slice_in(subchunk.len(), A::default()); + std::mem::MaybeUninit::write_slice(Arc::get_mut_unchecked(&mut slice), subchunk); + slice.assume_init() + }; result.push(Self { - storage: subchunk, + storage, _marker: std::marker::PhantomData, }); } @@ -295,7 +307,7 @@ pub(crate) struct GenericPolynomialChunk< P: field::traits::field_like::PrimeFieldLikeVectorized = F, A: GoodAllocator = Global, > { - pub(crate) over: std::sync::Arc>, + pub(crate) over: GenericPolynomial, pub(crate) range: std::ops::Range, _marker: std::marker::PhantomData<(FORM, A)>, } @@ -445,7 +457,9 @@ impl<'a, F: PrimeField, FORM: PolynomialForm, A: GoodAllocator, B: GoodAllocator let mut tmp: Vec<_> = polys .iter_mut() - .map(|el| el.storage.chunks_mut(chunk_size)) + .map(|el| { + unsafe { Arc::get_mut_unchecked(&mut el.storage) }.chunks_mut(chunk_size) + }) .collect(); for chunk_idx in 0..num_chunks { for poly_chunks in tmp.iter_mut() { diff --git a/src/cs/implementations/prover.rs b/src/cs/implementations/prover.rs index 2d2b4e3..505818f 100644 --- a/src/cs/implementations/prover.rs +++ b/src/cs/implementations/prover.rs @@ -1165,12 +1165,12 @@ impl< for el in quotient_c0s.into_iter() { q_c0_as_lde .storage - .push(Arc::new(GenericPolynomial::from_storage(el))); + .push(GenericPolynomial::from_storage(el)); } for el in quotient_c1s.into_iter() { q_c1_as_lde .storage - .push(Arc::new(GenericPolynomial::from_storage(el))); + .push(GenericPolynomial::from_storage(el)); } let mut challenges_it = remaining_challenges.iter(); @@ -1216,12 +1216,14 @@ impl< ); unsafe { - Arc::get_mut_unchecked(&mut dst[0].storage[outer]).storage[inner] - .add_assign(&z_c0, ctx); + Arc::get_mut_unchecked(&mut dst[0].storage[outer].storage)[inner] + } + .add_assign(&z_c0, ctx); - Arc::get_mut_unchecked(&mut dst[1].storage[outer]).storage[inner] - .add_assign(&z_c1, ctx); + unsafe { + Arc::get_mut_unchecked(&mut dst[1].storage[outer].storage)[inner] } + .add_assign(&z_c1, ctx); }; apply_multiop(&mut dst, &src, &op, worker, ctx) @@ -1366,21 +1368,13 @@ impl< let mut q_c0_as_vectors: Vec> = q_c0_as_lde .storage .into_iter() - .map(|el| { - Arc::try_unwrap(el) - .expect("must be exclusively owned") - .into_storage() - }) + .map(|el| el.into_storage()) .collect(); let mut q_c1_as_vectors: Vec> = q_c1_as_lde .storage .into_iter() - .map(|el| { - Arc::try_unwrap(el) - .expect("must be exclusively owned") - .into_storage() - }) + .map(|el| el.into_storage()) .collect(); if crate::config::DEBUG_SATISFIABLE == false { @@ -1552,7 +1546,7 @@ impl< .variables .variables_columns .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1561,7 +1555,7 @@ impl< .variables .witness_columns .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1571,7 +1565,7 @@ impl< .setup .constant_columns .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1580,7 +1574,7 @@ impl< .setup .copy_permutation_polys .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1588,10 +1582,8 @@ impl< all_polys_at_zs.push(ExtensionField::::from_coeff_in_base( evaluate_from_extension( &second_stage_polys_storage.z_poly[0].storage[0] - .as_ref() .storage, &second_stage_polys_storage.z_poly[1].storage[0] - .as_ref() .storage, ), )); @@ -1601,8 +1593,8 @@ impl< .iter() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -1625,7 +1617,7 @@ impl< .variables .lookup_multiplicities_polys .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1636,8 +1628,8 @@ impl< .iter() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -1649,8 +1641,8 @@ impl< .iter() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -1663,7 +1655,7 @@ impl< .setup .lookup_tables_columns .iter() - .map(|el| &el.storage[0].as_ref().storage) + .map(|el| &el.storage[0].storage) .map(|el| evaluate_from_base(el)) .map(|el| ExtensionField::::from_coeff_in_base(el)), ); @@ -1674,8 +1666,8 @@ impl< .array_chunks::<2>() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -1728,12 +1720,8 @@ impl< let all_polys_at_zomegas = vec![ExtensionField::::from_coeff_in_base( evaluate_from_extension( - &second_stage_polys_storage.z_poly[0].storage[0] - .as_ref() - .storage, - &second_stage_polys_storage.z_poly[1].storage[0] - .as_ref() - .storage, + &second_stage_polys_storage.z_poly[0].storage[0].storage, + &second_stage_polys_storage.z_poly[1].storage[0].storage, ), )]; assert_eq!(all_polys_at_zomegas.len(), 1); @@ -1773,8 +1761,8 @@ impl< .iter() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -1786,8 +1774,8 @@ impl< .iter() .map(|[a, b]| { [ - &a.storage[0].as_ref().storage, - &b.storage[0].as_ref().storage, + &a.storage[0].storage, + &b.storage[0].storage, ] }) .map(|[a, b]| evaluate_from_extension(a, b)) @@ -2816,8 +2804,8 @@ pub fn compute_selector_subpath< let mut tmp = one; tmp.sub_assign(&dst.storage[outer].storage[inner], &mut ctx); unsafe { - Arc::get_mut_unchecked(&mut dst.storage[outer]).storage[inner] = tmp - }; + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] = tmp; + } chunk.advance(); } }) @@ -2872,15 +2860,15 @@ pub fn compute_selector_subpath< result.sub_assign(&dst.storage[outer].storage[inner], &mut ctx); result.mul_assign(&prefix_poly.storage[outer].storage[inner], &mut ctx); unsafe { - Arc::get_mut_unchecked(&mut dst.storage[outer]).storage[inner] = result - }; + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] = result; + } } else { // we need prefix * this let mut result = dst.storage[outer].storage[inner]; result.mul_assign(&prefix_poly.storage[outer].storage[inner], &mut ctx); unsafe { - Arc::get_mut_unchecked(&mut dst.storage[outer]).storage[inner] = result - }; + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] = result; + } } chunk.advance(); diff --git a/src/cs/implementations/setup.rs b/src/cs/implementations/setup.rs index 104cd8e..e0c6766 100644 --- a/src/cs/implementations/setup.rs +++ b/src/cs/implementations/setup.rs @@ -53,8 +53,9 @@ fn materialize_x_by_non_residue_polys< let mut ctx = *ctx; scope.spawn(move |_| { for (non_res, poly) in non_residues.iter().zip(polys.iter_mut()) { + let poly_storage = unsafe { Arc::get_mut_unchecked(&mut poly.storage) }; let non_res = P::constant(*non_res, &mut ctx); - for el in poly.storage.iter_mut() { + for el in poly_storage.iter_mut() { el.mul_assign(&non_res, &mut ctx); } } @@ -449,7 +450,8 @@ impl< // we encountered this var before and stored it's PREVIOUS occurance // (read from poly) as field element, so we write previous occurance HERE, // and store this occurance as previous one - std::mem::swap(&mut previous_occurance_data.0, &mut poly.storage[row]); + let poly_storage = unsafe { Arc::get_mut_unchecked(&mut poly.storage) }; + std::mem::swap(&mut previous_occurance_data.0, &mut poly_storage[row]); } } } @@ -466,7 +468,8 @@ impl< } // worst case it reassign same, otherwise we reassign LAST occurance into first - result[first_encountered_column as usize].storage[first_encountered_row as usize] = + let result_storage = unsafe { Arc::get_mut_unchecked(&mut result[first_encountered_column as usize].storage) }; + result_storage[first_encountered_row as usize] = value; } @@ -923,10 +926,14 @@ impl< let content = table.content_at_row(row); debug_assert_eq!(content.len() + 1, result.len()); for (dst, src) in result[..content.len()].iter_mut().zip(content.iter()) { - dst.storage[idx] = *src; + unsafe { + Arc::get_mut_unchecked(&mut dst.storage)[idx] = *src; + } } let dst = &mut result[content.len()]; - dst.storage[idx] = table_id; + unsafe { + Arc::get_mut_unchecked(&mut dst.storage)[idx] = table_id; + } idx += 1; } } diff --git a/src/cs/implementations/utils.rs b/src/cs/implementations/utils.rs index af36492..f3847cd 100644 --- a/src/cs/implementations/utils.rs +++ b/src/cs/implementations/utils.rs @@ -394,7 +394,7 @@ pub(crate) fn transform_monomials_to_lde< let as_polynomial = GenericPolynomial::from_storage(el); columns[dst_poly_idx] .storage - .push(std::sync::Arc::new(as_polynomial)); + .push(as_polynomial); } // log!("{} LDEs of degree {} taken {:?}", num_polys, lde_degree, now.elapsed()); @@ -761,7 +761,7 @@ pub(crate) fn materialize_x_poly_as_arc_lde< result .storage - .push(std::sync::Arc::new(GenericPolynomial::from_storage(dst))); + .push(GenericPolynomial::from_storage(dst)); } result @@ -1315,10 +1315,9 @@ pub(crate) fn fused_multiply_sub< let mut r = a.storage[outer].storage[inner]; r.sub_assign(&t, &mut ctx); - unsafe { Arc::get_mut_unchecked(&mut result.storage[outer]) } - .storage - .spare_capacity_mut()[inner] - .write(r); + unsafe { + Arc::get_mut_unchecked(&mut result.storage[outer].storage)[inner] = r; + } iterator.advance(); } @@ -1326,7 +1325,7 @@ pub(crate) fn fused_multiply_sub< } }); - unsafe { result.assume_init(inner_size) }; + //unsafe { result.assume_init(inner_size) }; result } @@ -1370,9 +1369,11 @@ pub(crate) fn apply_binop_into< let b = &b.storage[outer].storage[inner]; // inliner should take care of references here + let mut dst = unsafe { + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] + }; op.apply( - &mut unsafe { Arc::get_mut_unchecked(&mut dst.storage[outer]) }.storage - [inner], + &mut dst, a, b, &mut ctx, @@ -1499,9 +1500,11 @@ pub(crate) fn apply_ternop_into< let c = &c.storage[outer].storage[inner]; // inliner should take care of references here + let mut dst = unsafe { + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] + }; op.apply( - &mut unsafe { Arc::get_mut_unchecked(&mut dst.storage[outer]) }.storage - [inner], + &mut dst, a, b, c, @@ -1564,9 +1567,11 @@ pub(crate) fn apply_quad_into< let d = &d.storage[outer].storage[inner]; // inliner should take care of references here + let mut dst = unsafe { + Arc::get_mut_unchecked(&mut dst.storage[outer].storage)[inner] + }; op.apply( - &mut unsafe { Arc::get_mut_unchecked(&mut dst.storage[outer]) }.storage - [inner], + &mut dst, a, b, c, @@ -1658,7 +1663,7 @@ pub(crate) fn unnormalized_l1_inverse< result .storage - .push(Arc::new(GenericPolynomial::from_storage(r))); + .push(GenericPolynomial::from_storage(r)); } result diff --git a/src/cs/implementations/witness.rs b/src/cs/implementations/witness.rs index be1f5e8..ff12605 100644 --- a/src/cs/implementations/witness.rs +++ b/src/cs/implementations/witness.rs @@ -146,7 +146,8 @@ impl< { scope.spawn(move |_| { for (dst, src) in dst.iter_mut().zip(src.iter()) { - for (dst, src) in dst.storage.iter_mut().zip(src.iter()) { + let dst_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut dst.storage) }; + for (dst, src) in dst_storage.iter_mut().zip(src.iter()) { if src.is_placeholder() == false { *dst = witness_ref[src.0 as usize]; } @@ -206,7 +207,8 @@ impl< scope.spawn(move |_| { debug_assert_eq!(vars_chunk.len(), polys_chunk.len()); for (vars_column, poly) in vars_chunk.iter().zip(polys_chunk.iter_mut()) { - for (var, dst) in vars_column.iter().zip(poly.storage.iter_mut()) { + let poly_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut poly.storage) }; + for (var, dst) in vars_column.iter().zip(poly_storage.iter_mut()) { if var.is_placeholder() == false { *dst = witness_ref[var.0 as usize]; } else { @@ -254,7 +256,8 @@ impl< let src_it = flattening_iter.clone().skip(num_to_skip); worker.scope(dst.storage.len(), |scope, chunk_size| { - for (idx, dst) in dst.storage.chunks_mut(chunk_size).enumerate() { + let dst_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut dst.storage) }; + for (idx, dst) in dst_storage.chunks_mut(chunk_size).enumerate() { let src = src_it.clone().skip(idx * chunk_size); scope.spawn(move |_| { for (dst, src) in dst.iter_mut().zip(src) { @@ -309,7 +312,8 @@ impl< { scope.spawn(move |_| { for (dst, src) in dst.iter_mut().zip(src.iter()) { - for (dst, src) in dst.storage.iter_mut().zip(src.iter()) { + let dst_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut dst.storage) }; + for (dst, src) in dst_storage.iter_mut().zip(src.iter()) { if src.is_placeholder() == false { *dst = witness_ref[src.0 as usize]; } @@ -368,7 +372,8 @@ impl< scope.spawn(move |_| { debug_assert_eq!(vars_chunk.len(), polys_chunk.len()); for (vars_column, poly) in vars_chunk.iter().zip(polys_chunk.iter_mut()) { - for (var, dst) in vars_column.iter().zip(poly.storage.iter_mut()) { + let poly_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut poly.storage) }; + for (var, dst) in vars_column.iter().zip(poly_storage.iter_mut()) { if var.is_placeholder() == false { *dst = witness_ref[var.0 as usize]; } else { @@ -425,7 +430,8 @@ impl< scope.spawn(move |_| { debug_assert_eq!(vars_chunk.len(), polys_chunk.len()); for (vars_column, poly) in vars_chunk.iter().zip(polys_chunk.iter_mut()) { - for (var, dst) in vars_column.iter().zip(poly.storage.iter_mut()) { + let poly_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut poly.storage) }; + for (var, dst) in vars_column.iter().zip(poly_storage.iter_mut()) { if var.is_placeholder() == false { // our index is just the index of the variable let as_usize = var.as_variable_index() as usize; @@ -472,7 +478,8 @@ impl< scope.spawn(move |_| { debug_assert_eq!(vars_chunk.len(), polys_chunk.len()); for (vars_column, poly) in vars_chunk.iter().zip(polys_chunk.iter_mut()) { - for (var, dst) in vars_column.iter().zip(poly.storage.iter_mut()) { + let poly_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut poly.storage) }; + for (var, dst) in vars_column.iter().zip(poly_storage.iter_mut()) { if var.is_placeholder() == false { // our index is just the index of the variable let as_usize = var.as_witness_index() as usize; @@ -507,8 +514,8 @@ impl< result.push(poly); // we know it's only 1 - for (dst, src) in result[0] - .storage + let result_storage = unsafe { std::sync::Arc::get_mut_unchecked(&mut result[0].storage) }; + for (dst, src) in result_storage .iter_mut() .zip(witness_set.multiplicities.iter().copied()) { diff --git a/src/lib.rs b/src/lib.rs index 72775d5..e92975f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,8 @@ #![allow(incomplete_features)] // Enabled features #![feature(allocator_api)] +#![feature(new_uninit)] +#![feature(maybe_uninit_write_slice)] #![feature(const_mut_refs)] #![feature(const_swap)] #![feature(inline_const)] diff --git a/src/utils.rs b/src/utils.rs index 4da61a6..25a2f63 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -6,6 +6,7 @@ use std::{ }; use derivative::Derivative; +use ecow::EcoVec; use crate::cs::traits::GoodAllocator; @@ -329,6 +330,130 @@ where seq.end() } +struct ArcSliceVisitor { + element: std::marker::PhantomData<(T, A)>, +} + +impl<'de, T, A: GoodAllocator> serde::de::Visitor<'de> for ArcSliceVisitor +where + T: serde::Deserialize<'de>, +{ + // FIXME: current version doesn't implement allocator API for Arc, check which one does + type Value = std::sync::Arc<[T], A>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a vector") + } + + fn visit_seq(self, mut seq: B) -> Result + where + B: serde::de::SeqAccess<'de>, + { + let expected_len = seq.size_hint().unwrap_or(0); + let mut slice = std::sync::Arc::new_uninit_slice_in(expected_len, A::default()); + let data = std::sync::Arc::get_mut(&mut slice).unwrap(); + for i in 0..expected_len { + let el = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?; + data[i].write(el); + } + let slice = unsafe { slice.assume_init() }; + + Ok(slice) + } +} + +pub fn deserialize_arc_slice<'de, D, T: serde::Deserialize<'de> + Clone, A: GoodAllocator>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let visitor = ArcSliceVisitor:: { + element: std::marker::PhantomData, + }; + deserializer.deserialize_seq(visitor) +} + +pub(crate) fn serialize_arc_slice( + t: &std::sync::Arc<[T], A>, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let mut seq = serializer.serialize_seq(Some(t.len()))?; + use serde::ser::SerializeSeq; + + for el in t.iter() { + let inner = &*el; + seq.serialize_element(inner)?; + } + seq.end() +} + +struct EcoVecVisitor { + element: std::marker::PhantomData, +} + +impl<'de, T> serde::de::Visitor<'de> for EcoVecVisitor +where + T: Clone + serde::Deserialize<'de>, +{ + type Value = EcoVec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a vector") + } + + fn visit_seq(self, mut seq: B) -> Result + where + B: serde::de::SeqAccess<'de>, + { + let expected_len = seq.size_hint().unwrap_or(0); + let mut vector = EcoVec::with_capacity(expected_len); + for i in 0..expected_len { + let el = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?; + vector.push(el); + } + + Ok(vector) + } +} + +pub fn deserialize_ecovec<'de, D, T: serde::Deserialize<'de> + Clone>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let visitor = EcoVecVisitor:: { + element: std::marker::PhantomData, + }; + deserializer.deserialize_seq(visitor) +} + +pub(crate) fn serialize_ecovec( + t: &EcoVec, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let mut seq = serializer.serialize_seq(Some(t.len()))?; + use serde::ser::SerializeSeq; + + for el in t.iter() { + let inner = &*el; + seq.serialize_element(inner)?; + } + seq.end() +} + + struct VecVisitor { element: std::marker::PhantomData<(T, A)>, } From f53381c1745999b89bdd7f893ccfe7213900b4ee Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Mon, 29 Apr 2024 15:24:07 -0300 Subject: [PATCH 2/3] Revert "fix modulus - 0 case" This reverts commit 19988079852ea22576da6b09e39365e6cdc1368f. Temporarily revert for testing, since this ensures I'm based on what era is running. --- .../implementations/implementation_u16.rs | 7 +-- .../non_native_field/implementations/utils.rs | 55 ------------------- 2 files changed, 3 insertions(+), 59 deletions(-) diff --git a/src/gadgets/non_native_field/implementations/implementation_u16.rs b/src/gadgets/non_native_field/implementations/implementation_u16.rs index f29509b..1074d04 100644 --- a/src/gadgets/non_native_field/implementations/implementation_u16.rs +++ b/src/gadgets/non_native_field/implementations/implementation_u16.rs @@ -122,7 +122,7 @@ where .map(|el| cs.allocate_constant(F::from_u64_unchecked(el as u64))); // for rare case when our modulus is exactly 16 * K bits, but we use larger representation let els_to_skip = N - self.params.modulus_limbs; - let _ = u16_long_subtraction_noborrow_must_borrow(cs, &self.limbs, &modulus, els_to_skip); + let _ = u16_long_subtraction_noborrow(cs, &modulus, &self.limbs, els_to_skip); self.tracker.max_moduluses = 1; } @@ -148,8 +148,7 @@ where .map(|el| cs.allocate_constant(F::from_u64_unchecked(el as u64))); // for rare case when our modulus is exactly 16 * K bits, but we use larger representation let els_to_skip = N - self.params.modulus_limbs; - let _ = - u16_long_subtraction_noborrow_must_borrow(cs, &normalized.limbs, &modulus, els_to_skip); + let _ = u16_long_subtraction_noborrow(cs, &modulus, &normalized.limbs, els_to_skip); assert!(normalized.form == RepresentationForm::Normalized); normalized.tracker.max_moduluses = 1; @@ -764,7 +763,7 @@ where let new = Self { limbs, non_zero_limbs: used_words, - tracker: OverflowTracker { max_moduluses: 2 }, // NOTE: if self == 0, then limbs will be == modulus, so use 2 + tracker: self.tracker, form: RepresentationForm::Normalized, params: self.params.clone(), _marker: std::marker::PhantomData, diff --git a/src/gadgets/non_native_field/implementations/utils.rs b/src/gadgets/non_native_field/implementations/utils.rs index 6d215b9..09f1721 100644 --- a/src/gadgets/non_native_field/implementations/utils.rs +++ b/src/gadgets/non_native_field/implementations/utils.rs @@ -185,61 +185,6 @@ pub fn u16_long_subtraction_noborrow, con result } -pub fn u16_long_subtraction_noborrow_must_borrow< - F: SmallField, - CS: ConstraintSystem, - const N: usize, ->( - cs: &mut CS, - a: &[Variable; N], - b: &[Variable; N], - top_els_to_skip: usize, -) -> [Variable; N] { - assert!(top_els_to_skip < N - 1); // at least some useful work - let constant_false = Boolean::allocated_constant(cs, false); - let constant_true = Boolean::allocated_constant(cs, true); - let mut borrow = constant_false; - let mut result = [constant_false.variable; N]; - let work_size = N - top_els_to_skip; - for (idx, ((a, b), dst)) in a[..work_size] - .iter() - .zip(b[..work_size].iter()) - .zip(result[..work_size].iter_mut()) - .enumerate() - { - let is_last = idx == work_size - 1; - if is_last == false { - // subtract with borrow - let (c, new_borrow) = - UIntXAddGate::<16>::perform_subtraction(cs, *a, *b, borrow.variable); - range_check_u16(cs, c); - *dst = c; - borrow = unsafe { Boolean::from_variable_unchecked(new_borrow) }; - } else { - // final one without borrow - let c = UIntXAddGate::<16>::perform_subtraction_with_expected_borrow_out( - cs, - *a, - *b, - borrow.variable, - constant_false.variable, - constant_true.variable, - ); - range_check_u16(cs, c); - *dst = c; - } - } - - let zero = Num::allocated_constant(cs, F::ZERO); - - for (a, b) in a[work_size..].iter().zip(b[work_size..].iter()) { - Num::enforce_equal(cs, &Num::from_variable(*a), &zero); - Num::enforce_equal(cs, &Num::from_variable(*b), &zero); - } - - result -} - pub fn split_out_u32_carry_from_zero_low>( cs: &mut CS, lhs: Variable, From 22811fcd1e506f3ee33601bb5fdc0d125f6e9441 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Mon, 29 Apr 2024 15:40:46 -0300 Subject: [PATCH 3/3] Manual clone --- src/cs/implementations/polynomial/lde.rs | 35 +++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/cs/implementations/polynomial/lde.rs b/src/cs/implementations/polynomial/lde.rs index 6878c1d..ea695d3 100644 --- a/src/cs/implementations/polynomial/lde.rs +++ b/src/cs/implementations/polynomial/lde.rs @@ -153,7 +153,7 @@ pub struct LdeParameters { } #[derive(Derivative, serde::Serialize, serde::Deserialize)] -#[derivative(Clone, Debug, PartialEq(bound = ""), Eq)] +#[derivative(Debug, PartialEq(bound = ""), Eq)] #[serde( bound = "F: serde::Serialize + serde::de::DeserializeOwned, P: serde::Serialize + serde::de::DeserializeOwned" )] @@ -168,6 +168,39 @@ pub struct ArcGenericLdeStorage< pub storage: Vec, B>, } +impl< + F: PrimeField, + P: field::traits::field_like::PrimeFieldLikeVectorized, + A: GoodAllocator, + B: GoodAllocator, +> Clone for ArcGenericLdeStorage { + #[inline(always)] + fn clone(&self) -> Self { + let mut storage = Vec::with_capacity_in(self.storage.len(), B::default()); + for el in self.storage.iter() { + // Explicitly clone storage to preserve `Arc` semantics + storage.push(GenericPolynomial { + storage: el.storage.clone(), + _marker: std::marker::PhantomData::default(), + }); + } + Self{storage} + } + + #[inline(always)] + fn clone_from(&mut self, other: &Self) { + self.storage.clear(); + self.storage.reserve(other.storage.len()); + for el in other.storage.iter() { + // Explicitly clone storage to preserve `Arc` semantics + self.storage.push(GenericPolynomial { + storage: el.storage.clone(), + _marker: std::marker::PhantomData::default(), + }); + } + } +} + pub type ArcLdeStorage = ArcGenericLdeStorage; impl<