From f0f4bf9a736fc7e96d0afe817361643bdd02d9cb Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Thu, 29 Aug 2024 12:33:16 +0530 Subject: [PATCH 1/9] separate workers --- Cargo.lock | 19 +++---------------- Cargo.toml | 3 +-- zero_bin/README.md | 2 +- zero_bin/leader/src/client.rs | 9 ++++++--- zero_bin/leader/src/http.rs | 27 ++++++++++++++++++++------- zero_bin/leader/src/main.rs | 30 +++++++++++++++++++++++++----- zero_bin/leader/src/stdio.rs | 17 +++++++++++++---- zero_bin/prover/src/lib.rs | 25 ++++++++++++++++--------- 8 files changed, 85 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b3eaf1d8..062983741 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3347,8 +3347,7 @@ dependencies = [ [[package]] name = "paladin-core" version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5af1955eaab1506a43d046628c218b7b3915539554838feb85ed31f54aace2f2" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#2ba9ac248eb22b4480b01d78c8182609bbdff0fe" dependencies = [ "anyhow", "async-trait", @@ -3366,6 +3365,7 @@ dependencies = [ "paladin-opkind-derive", "pin-project", "postcard", + "rand", "serde", "thiserror", "tokio", @@ -3374,14 +3374,12 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "uuid", ] [[package]] name = "paladin-opkind-derive" version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25dcb10b7c0ce99abee8694e2e79e4787d7f778b9339dc5a50ba6fc45e5cc9" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#2ba9ac248eb22b4480b01d78c8182609bbdff0fe" dependencies = [ "quote", "syn 2.0.70", @@ -5383,17 +5381,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" -dependencies = [ - "getrandom", - "rand", - "serde", -] - [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index cd48fcca2..c5b87de1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,7 +78,7 @@ num-bigint = "0.4.5" num-traits = "0.2.19" nunny = "0.2.1" once_cell = "1.19.0" -paladin-core = "0.4.2" +paladin-core = { git = "https://github.com/0xPolygonZero/paladin.git", branch = "arpit/507-2" } parking_lot = "0.12.3" paste = "1.0.15" pest = "2.7.10" @@ -139,4 +139,3 @@ trybuild = "1.0" [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" - diff --git a/zero_bin/README.md b/zero_bin/README.md index b6221a724..d8b03ce8f 100644 --- a/zero_bin/README.md +++ b/zero_bin/README.md @@ -101,7 +101,7 @@ Paladin options: -t, --task-bus-routing-key Specifies the routing key for publishing task messages. In most cases, the default value should suffice - [default: task] + [default: ""] -s, --serializer Determines the serialization format to be used diff --git a/zero_bin/leader/src/client.rs b/zero_bin/leader/src/client.rs index 4ef5ed3cd..d110d2a38 100644 --- a/zero_bin/leader/src/client.rs +++ b/zero_bin/leader/src/client.rs @@ -33,7 +33,8 @@ pub struct ProofParams { /// The main function for the client. pub(crate) async fn client_main( - runtime: Runtime, + block_proof_runtime: Runtime, + segment_runtime: Runtime, rpc_params: RpcParams, block_interval: BlockInterval, mut params: ProofParams, @@ -82,13 +83,15 @@ pub(crate) async fn client_main( // verify the whole sequence. let proved_blocks = prover::prove( block_prover_inputs, - &runtime, + &block_proof_runtime, + &segment_runtime, params.previous_proof.take(), params.prover_config, params.proof_output_dir.clone(), ) .await; - runtime.close().await?; + block_proof_runtime.close().await?; + segment_runtime.close().await?; let proved_blocks = proved_blocks?; if params.prover_config.test_only { diff --git a/zero_bin/leader/src/http.rs b/zero_bin/leader/src/http.rs index 39c7333e1..d05e9252f 100644 --- a/zero_bin/leader/src/http.rs +++ b/zero_bin/leader/src/http.rs @@ -12,7 +12,8 @@ use tracing::{debug, error, info}; /// The main function for the HTTP mode. pub(crate) async fn http_main( - runtime: Runtime, + block_proof_runtime: Runtime, + segment_runtime: Runtime, port: u16, output_dir: PathBuf, prover_config: ProverConfig, @@ -20,12 +21,22 @@ pub(crate) async fn http_main( let addr = SocketAddr::from(([0, 0, 0, 0], port)); debug!("listening on {}", addr); - let runtime = Arc::new(runtime); + let block_proof_runtime = Arc::new(block_proof_runtime); + let segment_runtime = Arc::new(segment_runtime); let app = Router::new().route( "/prove", post({ - let runtime = runtime.clone(); - move |body| prove(body, runtime, output_dir.clone(), prover_config) + let block_proof_runtime = block_proof_runtime.clone(); + let segment_runtime = segment_runtime.clone(); + move |body| { + prove( + body, + block_proof_runtime, + segment_runtime, + output_dir.clone(), + prover_config, + ) + } }), ); let listener = tokio::net::TcpListener::bind(&addr).await?; @@ -63,7 +74,8 @@ struct HttpProverInput { async fn prove( Json(payload): Json, - runtime: Arc, + block_proof_runtime: Arc, + segment_runtime: Arc, output_dir: PathBuf, prover_config: ProverConfig, ) -> StatusCode { @@ -75,7 +87,7 @@ async fn prove( payload .prover_input .prove_test( - &runtime, + &segment_runtime, payload.previous.map(futures::future::ok), prover_config, ) @@ -84,7 +96,8 @@ async fn prove( payload .prover_input .prove( - &runtime, + &block_proof_runtime, + &segment_runtime, payload.previous.map(futures::future::ok), prover_config, ) diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index f4a448a3b..e466d7c4b 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -55,7 +55,14 @@ async fn main() -> Result<()> { let args = cli::Cli::parse(); - let runtime = Runtime::from_config(&args.paladin, register()).await?; + let mut block_proof_paladin_args = args.paladin.clone(); + block_proof_paladin_args.task_bus_routing_key = Some("block_proof".to_string()); + + let mut segment_paladin_args = args.paladin.clone(); + segment_paladin_args.task_bus_routing_key = Some("segment".to_string()); + + let block_proof_runtime = Runtime::from_config(&block_proof_paladin_args, register()).await?; + let segment_runtime = Runtime::from_config(&segment_paladin_args, register()).await?; let prover_config: ProverConfig = args.prover_config.into(); @@ -73,7 +80,13 @@ async fn main() -> Result<()> { Command::Clean => zero_bin_common::prover_state::persistence::delete_all()?, Command::Stdio { previous_proof } => { let previous_proof = get_previous_proof(previous_proof)?; - stdio::stdio_main(runtime, previous_proof, prover_config).await?; + stdio::stdio_main( + block_proof_runtime, + segment_runtime, + previous_proof, + prover_config, + ) + .await?; } Command::Http { port, output_dir } => { // check if output_dir exists, is a directory, and is writable @@ -85,7 +98,14 @@ async fn main() -> Result<()> { panic!("output-dir is not a writable directory"); } - http::http_main(runtime, port, output_dir, prover_config).await?; + http::http_main( + block_proof_runtime, + segment_runtime, + port, + output_dir, + prover_config, + ) + .await?; } Command::Rpc { rpc_url, @@ -99,7 +119,6 @@ async fn main() -> Result<()> { backoff, max_retries, } => { - let runtime = Runtime::from_config(&args.paladin, register()).await?; let previous_proof = get_previous_proof(previous_proof)?; let mut block_interval = BlockInterval::new(&block_interval)?; @@ -113,7 +132,8 @@ async fn main() -> Result<()> { info!("Proving interval {block_interval}"); client_main( - runtime, + block_proof_runtime, + segment_runtime, RpcParams { rpc_url, rpc_type, diff --git a/zero_bin/leader/src/stdio.rs b/zero_bin/leader/src/stdio.rs index 88dd20aac..ef9392564 100644 --- a/zero_bin/leader/src/stdio.rs +++ b/zero_bin/leader/src/stdio.rs @@ -8,7 +8,8 @@ use tracing::info; /// The main function for the stdio mode. pub(crate) async fn stdio_main( - runtime: Runtime, + block_proof_runtime: Runtime, + segment_runtime: Runtime, previous: Option, prover_config: ProverConfig, ) -> Result<()> { @@ -21,9 +22,17 @@ pub(crate) async fn stdio_main( .map(Into::into) .collect::>(); - let proved_blocks = - prover::prove(block_prover_inputs, &runtime, previous, prover_config, None).await; - runtime.close().await?; + let proved_blocks = prover::prove( + block_prover_inputs, + &block_proof_runtime, + &segment_runtime, + previous, + prover_config, + None, + ) + .await; + block_proof_runtime.close().await?; + segment_runtime.close().await?; let proved_blocks = proved_blocks?; if prover_config.test_only { diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index 46472bc7f..cb80bdccb 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -50,7 +50,8 @@ impl BlockProverInput { pub async fn prove( self, - runtime: &Runtime, + block_proof_runtime: &Runtime, + segment_runtime: &Runtime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -99,7 +100,7 @@ impl BlockProverInput { Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) .fold(&seg_agg_ops) - .run(runtime) + .run(segment_runtime) .map(move |e| { e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) }) @@ -109,7 +110,7 @@ impl BlockProverInput { // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) - .run(runtime) + .run(block_proof_runtime) .await?; if let proof_gen::proof_types::BatchAggregatableProof::Agg(proof) = final_batch_proof { @@ -126,7 +127,7 @@ impl BlockProverInput { prev, save_inputs_on_error, }) - .run(runtime) + .run(block_proof_runtime) .await?; info!("Successfully proved block {block_number}"); @@ -139,7 +140,7 @@ impl BlockProverInput { pub async fn prove_test( self, - runtime: &Runtime, + segment_runtime: &Runtime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -175,7 +176,7 @@ impl BlockProverInput { ); simulation - .run(runtime) + .run(segment_runtime) .await? .try_for_each(|_| future::ok(())) .await?; @@ -204,7 +205,8 @@ impl BlockProverInput { /// block proofs as well. pub async fn prove( block_prover_inputs: Vec, - runtime: &Runtime, + block_proof_runtime: &Runtime, + segment_runtime: &Runtime, previous_proof: Option, prover_config: ProverConfig, proof_output_dir: Option, @@ -226,7 +228,7 @@ pub async fn prove( // Prove the block let block_proof = if prover_config.test_only { block - .prove_test(runtime, previous_block_proof, prover_config) + .prove_test(segment_runtime, previous_block_proof, prover_config) .then(move |proof| async move { let proof = proof?; let block_number = proof.b_height; @@ -250,7 +252,12 @@ pub async fn prove( .await? } else { block - .prove(runtime, previous_block_proof, prover_config) + .prove( + block_proof_runtime, + segment_runtime, + previous_block_proof, + prover_config, + ) .then(move |proof| async move { let proof = proof?; let block_number = proof.b_height; From ee502950e2dc33de8cb374036a80b2784a085476 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Fri, 30 Aug 2024 17:08:03 +0530 Subject: [PATCH 2/9] fix review comments --- zero_bin/README.md | 2 +- zero_bin/leader/src/client.rs | 6 +++--- zero_bin/leader/src/http.rs | 14 +++++++------- zero_bin/leader/src/main.rs | 18 +++++++++++------- zero_bin/leader/src/stdio.rs | 6 +++--- zero_bin/prover/src/lib.rs | 14 +++++++------- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/zero_bin/README.md b/zero_bin/README.md index d8b03ce8f..b6221a724 100644 --- a/zero_bin/README.md +++ b/zero_bin/README.md @@ -101,7 +101,7 @@ Paladin options: -t, --task-bus-routing-key Specifies the routing key for publishing task messages. In most cases, the default value should suffice - [default: ""] + [default: task] -s, --serializer Determines the serialization format to be used diff --git a/zero_bin/leader/src/client.rs b/zero_bin/leader/src/client.rs index d110d2a38..62ac4afee 100644 --- a/zero_bin/leader/src/client.rs +++ b/zero_bin/leader/src/client.rs @@ -34,7 +34,7 @@ pub struct ProofParams { /// The main function for the client. pub(crate) async fn client_main( block_proof_runtime: Runtime, - segment_runtime: Runtime, + segment_proof_runtime: Runtime, rpc_params: RpcParams, block_interval: BlockInterval, mut params: ProofParams, @@ -84,14 +84,14 @@ pub(crate) async fn client_main( let proved_blocks = prover::prove( block_prover_inputs, &block_proof_runtime, - &segment_runtime, + &segment_proof_runtime, params.previous_proof.take(), params.prover_config, params.proof_output_dir.clone(), ) .await; block_proof_runtime.close().await?; - segment_runtime.close().await?; + segment_proof_runtime.close().await?; let proved_blocks = proved_blocks?; if params.prover_config.test_only { diff --git a/zero_bin/leader/src/http.rs b/zero_bin/leader/src/http.rs index d05e9252f..fcde1b034 100644 --- a/zero_bin/leader/src/http.rs +++ b/zero_bin/leader/src/http.rs @@ -13,7 +13,7 @@ use tracing::{debug, error, info}; /// The main function for the HTTP mode. pub(crate) async fn http_main( block_proof_runtime: Runtime, - segment_runtime: Runtime, + segment_proof_runtime: Runtime, port: u16, output_dir: PathBuf, prover_config: ProverConfig, @@ -22,17 +22,17 @@ pub(crate) async fn http_main( debug!("listening on {}", addr); let block_proof_runtime = Arc::new(block_proof_runtime); - let segment_runtime = Arc::new(segment_runtime); + let segment_proof_runtime = Arc::new(segment_proof_runtime); let app = Router::new().route( "/prove", post({ let block_proof_runtime = block_proof_runtime.clone(); - let segment_runtime = segment_runtime.clone(); + let segment_proof_runtime = segment_proof_runtime.clone(); move |body| { prove( body, block_proof_runtime, - segment_runtime, + segment_proof_runtime, output_dir.clone(), prover_config, ) @@ -75,7 +75,7 @@ struct HttpProverInput { async fn prove( Json(payload): Json, block_proof_runtime: Arc, - segment_runtime: Arc, + segment_proof_runtime: Arc, output_dir: PathBuf, prover_config: ProverConfig, ) -> StatusCode { @@ -87,7 +87,7 @@ async fn prove( payload .prover_input .prove_test( - &segment_runtime, + &segment_proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) @@ -97,7 +97,7 @@ async fn prove( .prover_input .prove( &block_proof_runtime, - &segment_runtime, + &segment_proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index e466d7c4b..2d5bafbc7 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -36,6 +36,9 @@ fn get_previous_proof(path: Option) -> Result Result<()> { load_dotenvy_vars_if_present(); @@ -56,13 +59,14 @@ async fn main() -> Result<()> { let args = cli::Cli::parse(); let mut block_proof_paladin_args = args.paladin.clone(); - block_proof_paladin_args.task_bus_routing_key = Some("block_proof".to_string()); + block_proof_paladin_args.task_bus_routing_key = Some(BLOCK_PROOF_ROUTING_KEY.to_string()); - let mut segment_paladin_args = args.paladin.clone(); - segment_paladin_args.task_bus_routing_key = Some("segment".to_string()); + let mut segment_proof_paladin_args = args.paladin.clone(); + segment_proof_paladin_args.task_bus_routing_key = Some(SEGMENT_PROOF_ROUTING_KEY.to_string()); let block_proof_runtime = Runtime::from_config(&block_proof_paladin_args, register()).await?; - let segment_runtime = Runtime::from_config(&segment_paladin_args, register()).await?; + let segment_proof_runtime = + Runtime::from_config(&segment_proof_paladin_args, register()).await?; let prover_config: ProverConfig = args.prover_config.into(); @@ -82,7 +86,7 @@ async fn main() -> Result<()> { let previous_proof = get_previous_proof(previous_proof)?; stdio::stdio_main( block_proof_runtime, - segment_runtime, + segment_proof_runtime, previous_proof, prover_config, ) @@ -100,7 +104,7 @@ async fn main() -> Result<()> { http::http_main( block_proof_runtime, - segment_runtime, + segment_proof_runtime, port, output_dir, prover_config, @@ -133,7 +137,7 @@ async fn main() -> Result<()> { info!("Proving interval {block_interval}"); client_main( block_proof_runtime, - segment_runtime, + segment_proof_runtime, RpcParams { rpc_url, rpc_type, diff --git a/zero_bin/leader/src/stdio.rs b/zero_bin/leader/src/stdio.rs index ef9392564..3971a39f1 100644 --- a/zero_bin/leader/src/stdio.rs +++ b/zero_bin/leader/src/stdio.rs @@ -9,7 +9,7 @@ use tracing::info; /// The main function for the stdio mode. pub(crate) async fn stdio_main( block_proof_runtime: Runtime, - segment_runtime: Runtime, + segment_proof_runtime: Runtime, previous: Option, prover_config: ProverConfig, ) -> Result<()> { @@ -25,14 +25,14 @@ pub(crate) async fn stdio_main( let proved_blocks = prover::prove( block_prover_inputs, &block_proof_runtime, - &segment_runtime, + &segment_proof_runtime, previous, prover_config, None, ) .await; block_proof_runtime.close().await?; - segment_runtime.close().await?; + segment_proof_runtime.close().await?; let proved_blocks = proved_blocks?; if prover_config.test_only { diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index cb80bdccb..e98f9facd 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -51,7 +51,7 @@ impl BlockProverInput { pub async fn prove( self, block_proof_runtime: &Runtime, - segment_runtime: &Runtime, + segment_proof_runtime: &Runtime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -100,7 +100,7 @@ impl BlockProverInput { Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) .fold(&seg_agg_ops) - .run(segment_runtime) + .run(segment_proof_runtime) .map(move |e| { e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) }) @@ -140,7 +140,7 @@ impl BlockProverInput { pub async fn prove_test( self, - segment_runtime: &Runtime, + segment_proof_runtime: &Runtime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -176,7 +176,7 @@ impl BlockProverInput { ); simulation - .run(segment_runtime) + .run(segment_proof_runtime) .await? .try_for_each(|_| future::ok(())) .await?; @@ -206,7 +206,7 @@ impl BlockProverInput { pub async fn prove( block_prover_inputs: Vec, block_proof_runtime: &Runtime, - segment_runtime: &Runtime, + segment_proof_runtime: &Runtime, previous_proof: Option, prover_config: ProverConfig, proof_output_dir: Option, @@ -228,7 +228,7 @@ pub async fn prove( // Prove the block let block_proof = if prover_config.test_only { block - .prove_test(segment_runtime, previous_block_proof, prover_config) + .prove_test(segment_proof_runtime, previous_block_proof, prover_config) .then(move |proof| async move { let proof = proof?; let block_number = proof.b_height; @@ -254,7 +254,7 @@ pub async fn prove( block .prove( block_proof_runtime, - segment_runtime, + segment_proof_runtime, previous_block_proof, prover_config, ) From e941b569e9076d6be31f4c077affe1644e09e470 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 2 Sep 2024 13:29:51 +0530 Subject: [PATCH 3/9] add worker run modes --- zero_bin/leader/src/cli.rs | 14 +++++++++++++- zero_bin/leader/src/main.rs | 15 +++++++++++++-- zero_bin/tools/prove_rpc.sh | 4 ++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/zero_bin/leader/src/cli.rs b/zero_bin/leader/src/cli.rs index 50c6e7ce1..f05e45fae 100644 --- a/zero_bin/leader/src/cli.rs +++ b/zero_bin/leader/src/cli.rs @@ -1,11 +1,13 @@ use std::path::PathBuf; use alloy::transports::http::reqwest::Url; -use clap::{Parser, Subcommand, ValueHint}; +use clap::{Parser, Subcommand, ValueEnum, ValueHint}; use prover::cli::CliProverConfig; use rpc::RpcType; use zero_bin_common::prover_state::cli::CliProverStateConfig; +const WORKER_HELP_HEADING: &str = "Worker Config options"; + /// zero-bin leader config #[derive(Parser)] pub(crate) struct Cli { @@ -22,6 +24,16 @@ pub(crate) struct Cli { // mode. #[clap(flatten)] pub(crate) prover_state_config: CliProverStateConfig, + + // Mode to use for worker for setup (split or unified) + #[arg(long = "worker-run-mode", help_heading = WORKER_HELP_HEADING, value_enum, default_value = "unified")] + pub(crate) worker_run_mode: WorkerRunMode, +} + +#[derive(ValueEnum, Clone, PartialEq, Debug)] +pub enum WorkerRunMode { + Split, + Unified, } #[derive(Subcommand)] diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index 2d5bafbc7..f5ab19f81 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -38,6 +38,7 @@ fn get_previous_proof(path: Option) -> Result Result<()> { @@ -58,11 +59,21 @@ async fn main() -> Result<()> { let args = cli::Cli::parse(); + let mut block_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + let mut segment_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + if args.worker_run_mode == cli::WorkerRunMode::Split { + // If we're running in split mode, we need to set the routing key for the + // block proof and segment proof. + info!("Workers running in split mode"); + block_proof_routing_key = BLOCK_PROOF_ROUTING_KEY.to_string(); + segment_proof_routing_key = SEGMENT_PROOF_ROUTING_KEY.to_string(); + } + let mut block_proof_paladin_args = args.paladin.clone(); - block_proof_paladin_args.task_bus_routing_key = Some(BLOCK_PROOF_ROUTING_KEY.to_string()); + block_proof_paladin_args.task_bus_routing_key = Some(block_proof_routing_key); let mut segment_proof_paladin_args = args.paladin.clone(); - segment_proof_paladin_args.task_bus_routing_key = Some(SEGMENT_PROOF_ROUTING_KEY.to_string()); + segment_proof_paladin_args.task_bus_routing_key = Some(segment_proof_routing_key); let block_proof_runtime = Runtime::from_config(&block_proof_paladin_args, register()).await?; let segment_proof_runtime = diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 941521a3c..0409e399b 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -102,7 +102,7 @@ fi if [[ $8 == "test_only" ]]; then # test only run echo "Proving blocks ${BLOCK_INTERVAL} in a test_only mode now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --runtime in-memory --test-only --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command retVal=$? @@ -125,7 +125,7 @@ if [[ $8 == "test_only" ]]; then else # normal run echo "Proving blocks ${BLOCK_INTERVAL} now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES"' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command echo -e "Proof generation finished with result: $?" From 45cfeeb12e75651cbae8badcdb8eda14f6b865a0 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 2 Sep 2024 18:07:55 +0530 Subject: [PATCH 4/9] change default queue --- zero_bin/leader/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index f5ab19f81..775fabe6f 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -38,7 +38,7 @@ fn get_previous_proof(path: Option) -> Result Result<()> { From 8b00d3fb8df086f5c662630fdfd99f4824381a4e Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 2 Sep 2024 18:11:59 +0530 Subject: [PATCH 5/9] update cargo.lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 062983741..f1cbfe274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3347,7 +3347,7 @@ dependencies = [ [[package]] name = "paladin-core" version = "0.4.2" -source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#2ba9ac248eb22b4480b01d78c8182609bbdff0fe" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#6c752fd2ec9bc81dc2f4b727580d7dd7a42b9165" dependencies = [ "anyhow", "async-trait", @@ -3379,7 +3379,7 @@ dependencies = [ [[package]] name = "paladin-opkind-derive" version = "0.4.1" -source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#2ba9ac248eb22b4480b01d78c8182609bbdff0fe" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#6c752fd2ec9bc81dc2f4b727580d7dd7a42b9165" dependencies = [ "quote", "syn 2.0.70", From a994162c4c9cffaa270a6cc881fca9d26cc74633 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 3 Sep 2024 17:13:28 +0530 Subject: [PATCH 6/9] fix review comments --- Cargo.lock | 1 + zero_bin/common/Cargo.toml | 2 +- zero_bin/common/src/lib.rs | 1 + zero_bin/common/src/proof_runtime.rs | 6 ++++++ zero_bin/leader/src/client.rs | 12 +++++------ zero_bin/leader/src/http.rs | 30 +++++++--------------------- zero_bin/leader/src/main.rs | 28 ++++++++++---------------- zero_bin/leader/src/stdio.rs | 12 +++++------ zero_bin/prover/src/lib.rs | 28 ++++++++++---------------- zero_bin/tools/prove_rpc.sh | 4 ++-- 10 files changed, 48 insertions(+), 76 deletions(-) create mode 100644 zero_bin/common/src/proof_runtime.rs diff --git a/Cargo.lock b/Cargo.lock index f1cbfe274..3be6f5edc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5845,6 +5845,7 @@ dependencies = [ "evm_arithmetization", "futures", "once_cell", + "paladin-core", "plonky2", "proof_gen", "serde", diff --git a/zero_bin/common/Cargo.toml b/zero_bin/common/Cargo.toml index bfad3e2a8..a02d99b0c 100644 --- a/zero_bin/common/Cargo.toml +++ b/zero_bin/common/Cargo.toml @@ -10,7 +10,7 @@ categories.workspace = true [dependencies] directories = "5.0.1" - +paladin-core = { workspace = true } thiserror = { workspace = true } trace_decoder = { workspace = true } tracing = { workspace = true } diff --git a/zero_bin/common/src/lib.rs b/zero_bin/common/src/lib.rs index 4784cb6c4..9d379d632 100644 --- a/zero_bin/common/src/lib.rs +++ b/zero_bin/common/src/lib.rs @@ -3,5 +3,6 @@ pub mod debug_utils; pub mod fs; pub mod parsing; pub mod pre_checks; +pub mod proof_runtime; pub mod prover_state; pub mod version; diff --git a/zero_bin/common/src/proof_runtime.rs b/zero_bin/common/src/proof_runtime.rs new file mode 100644 index 000000000..b343b0ae9 --- /dev/null +++ b/zero_bin/common/src/proof_runtime.rs @@ -0,0 +1,6 @@ +use paladin::runtime::Runtime; + +pub struct ProofRuntime { + pub block_proof_runtime: Runtime, + pub segment_proof_runtime: Runtime, +} diff --git a/zero_bin/leader/src/client.rs b/zero_bin/leader/src/client.rs index 62ac4afee..053e31543 100644 --- a/zero_bin/leader/src/client.rs +++ b/zero_bin/leader/src/client.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use alloy::rpc::types::{BlockId, BlockNumberOrTag, BlockTransactionsKind}; use alloy::transports::http::reqwest::Url; use anyhow::Result; -use paladin::runtime::Runtime; use proof_gen::proof_types::GeneratedBlockProof; use prover::ProverConfig; use rpc::{retry::build_http_retry_provider, RpcType}; @@ -13,6 +12,7 @@ use tracing::{error, info, warn}; use zero_bin_common::block_interval::BlockInterval; use zero_bin_common::fs::generate_block_proof_file_name; use zero_bin_common::pre_checks::check_previous_proof_and_checkpoint; +use zero_bin_common::proof_runtime::ProofRuntime; #[derive(Debug)] pub struct RpcParams { @@ -33,8 +33,7 @@ pub struct ProofParams { /// The main function for the client. pub(crate) async fn client_main( - block_proof_runtime: Runtime, - segment_proof_runtime: Runtime, + proof_runtime: ProofRuntime, rpc_params: RpcParams, block_interval: BlockInterval, mut params: ProofParams, @@ -83,15 +82,14 @@ pub(crate) async fn client_main( // verify the whole sequence. let proved_blocks = prover::prove( block_prover_inputs, - &block_proof_runtime, - &segment_proof_runtime, + &proof_runtime, params.previous_proof.take(), params.prover_config, params.proof_output_dir.clone(), ) .await; - block_proof_runtime.close().await?; - segment_proof_runtime.close().await?; + proof_runtime.block_proof_runtime.close().await?; + proof_runtime.segment_proof_runtime.close().await?; let proved_blocks = proved_blocks?; if params.prover_config.test_only { diff --git a/zero_bin/leader/src/http.rs b/zero_bin/leader/src/http.rs index fcde1b034..913ac2833 100644 --- a/zero_bin/leader/src/http.rs +++ b/zero_bin/leader/src/http.rs @@ -3,17 +3,16 @@ use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use alloy::primitives::U256; use anyhow::{bail, Result}; use axum::{http::StatusCode, routing::post, Json, Router}; -use paladin::runtime::Runtime; use proof_gen::proof_types::GeneratedBlockProof; use prover::{BlockProverInput, ProverConfig}; use serde::{Deserialize, Serialize}; use serde_json::to_writer; use tracing::{debug, error, info}; +use zero_bin_common::proof_runtime::ProofRuntime; /// The main function for the HTTP mode. pub(crate) async fn http_main( - block_proof_runtime: Runtime, - segment_proof_runtime: Runtime, + proof_runtime: ProofRuntime, port: u16, output_dir: PathBuf, prover_config: ProverConfig, @@ -21,23 +20,10 @@ pub(crate) async fn http_main( let addr = SocketAddr::from(([0, 0, 0, 0], port)); debug!("listening on {}", addr); - let block_proof_runtime = Arc::new(block_proof_runtime); - let segment_proof_runtime = Arc::new(segment_proof_runtime); + let proof_runtime = Arc::new(proof_runtime); let app = Router::new().route( "/prove", - post({ - let block_proof_runtime = block_proof_runtime.clone(); - let segment_proof_runtime = segment_proof_runtime.clone(); - move |body| { - prove( - body, - block_proof_runtime, - segment_proof_runtime, - output_dir.clone(), - prover_config, - ) - } - }), + post(move |body| prove(body, proof_runtime, output_dir.clone(), prover_config)), ); let listener = tokio::net::TcpListener::bind(&addr).await?; Ok(axum::serve(listener, app).await?) @@ -74,8 +60,7 @@ struct HttpProverInput { async fn prove( Json(payload): Json, - block_proof_runtime: Arc, - segment_proof_runtime: Arc, + proof_runtime: Arc, output_dir: PathBuf, prover_config: ProverConfig, ) -> StatusCode { @@ -87,7 +72,7 @@ async fn prove( payload .prover_input .prove_test( - &segment_proof_runtime, + &proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) @@ -96,8 +81,7 @@ async fn prove( payload .prover_input .prove( - &block_proof_runtime, - &segment_proof_runtime, + &proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index 775fabe6f..582791854 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -12,7 +12,8 @@ use proof_gen::proof_types::GeneratedBlockProof; use prover::ProverConfig; use tracing::{info, warn}; use zero_bin_common::{ - block_interval::BlockInterval, prover_state::persistence::set_circuit_cache_dir_env_if_not_set, + block_interval::BlockInterval, proof_runtime::ProofRuntime, + prover_state::persistence::set_circuit_cache_dir_env_if_not_set, }; use zero_bin_common::{prover_state::persistence::CIRCUIT_VERSION, version}; @@ -91,17 +92,16 @@ async fn main() -> Result<()> { } } + let proof_runtime = ProofRuntime { + block_proof_runtime, + segment_proof_runtime, + }; + match args.command { Command::Clean => zero_bin_common::prover_state::persistence::delete_all()?, Command::Stdio { previous_proof } => { let previous_proof = get_previous_proof(previous_proof)?; - stdio::stdio_main( - block_proof_runtime, - segment_proof_runtime, - previous_proof, - prover_config, - ) - .await?; + stdio::stdio_main(proof_runtime, previous_proof, prover_config).await?; } Command::Http { port, output_dir } => { // check if output_dir exists, is a directory, and is writable @@ -113,14 +113,7 @@ async fn main() -> Result<()> { panic!("output-dir is not a writable directory"); } - http::http_main( - block_proof_runtime, - segment_proof_runtime, - port, - output_dir, - prover_config, - ) - .await?; + http::http_main(proof_runtime, port, output_dir, prover_config).await?; } Command::Rpc { rpc_url, @@ -147,8 +140,7 @@ async fn main() -> Result<()> { info!("Proving interval {block_interval}"); client_main( - block_proof_runtime, - segment_proof_runtime, + proof_runtime, RpcParams { rpc_url, rpc_type, diff --git a/zero_bin/leader/src/stdio.rs b/zero_bin/leader/src/stdio.rs index 3971a39f1..62644797b 100644 --- a/zero_bin/leader/src/stdio.rs +++ b/zero_bin/leader/src/stdio.rs @@ -1,15 +1,14 @@ use std::io::{Read, Write}; use anyhow::Result; -use paladin::runtime::Runtime; use proof_gen::proof_types::GeneratedBlockProof; use prover::{BlockProverInput, BlockProverInputFuture, ProverConfig}; use tracing::info; +use zero_bin_common::proof_runtime::ProofRuntime; /// The main function for the stdio mode. pub(crate) async fn stdio_main( - block_proof_runtime: Runtime, - segment_proof_runtime: Runtime, + proof_runtime: ProofRuntime, previous: Option, prover_config: ProverConfig, ) -> Result<()> { @@ -24,15 +23,14 @@ pub(crate) async fn stdio_main( let proved_blocks = prover::prove( block_prover_inputs, - &block_proof_runtime, - &segment_proof_runtime, + &proof_runtime, previous, prover_config, None, ) .await; - block_proof_runtime.close().await?; - segment_proof_runtime.close().await?; + proof_runtime.block_proof_runtime.close().await?; + proof_runtime.segment_proof_runtime.close().await?; let proved_blocks = proved_blocks?; if prover_config.test_only { diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index e98f9facd..6969821ad 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -7,14 +7,13 @@ use alloy::primitives::{BlockNumber, U256}; use anyhow::{Context, Result}; use futures::{future::BoxFuture, stream::FuturesOrdered, FutureExt, TryFutureExt, TryStreamExt}; use num_traits::ToPrimitive as _; -use paladin::runtime::Runtime; use proof_gen::proof_types::GeneratedBlockProof; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use tokio::sync::oneshot; use trace_decoder::{BlockTrace, OtherBlockData}; use tracing::info; -use zero_bin_common::fs::generate_block_proof_file_name; +use zero_bin_common::{fs::generate_block_proof_file_name, proof_runtime::ProofRuntime}; #[derive(Debug, Clone, Copy)] pub struct ProverConfig { @@ -50,8 +49,7 @@ impl BlockProverInput { pub async fn prove( self, - block_proof_runtime: &Runtime, - segment_proof_runtime: &Runtime, + proof_runtime: &ProofRuntime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -100,7 +98,7 @@ impl BlockProverInput { Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) .fold(&seg_agg_ops) - .run(segment_proof_runtime) + .run(&proof_runtime.segment_proof_runtime) .map(move |e| { e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) }) @@ -110,7 +108,7 @@ impl BlockProverInput { // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) - .run(block_proof_runtime) + .run(&proof_runtime.block_proof_runtime) .await?; if let proof_gen::proof_types::BatchAggregatableProof::Agg(proof) = final_batch_proof { @@ -127,7 +125,7 @@ impl BlockProverInput { prev, save_inputs_on_error, }) - .run(block_proof_runtime) + .run(&proof_runtime.block_proof_runtime) .await?; info!("Successfully proved block {block_number}"); @@ -140,7 +138,7 @@ impl BlockProverInput { pub async fn prove_test( self, - segment_proof_runtime: &Runtime, + proof_runtime: &ProofRuntime, previous: Option>>, prover_config: ProverConfig, ) -> Result { @@ -176,7 +174,7 @@ impl BlockProverInput { ); simulation - .run(segment_proof_runtime) + .run(&proof_runtime.segment_proof_runtime) .await? .try_for_each(|_| future::ok(())) .await?; @@ -205,8 +203,7 @@ impl BlockProverInput { /// block proofs as well. pub async fn prove( block_prover_inputs: Vec, - block_proof_runtime: &Runtime, - segment_proof_runtime: &Runtime, + proof_runtime: &ProofRuntime, previous_proof: Option, prover_config: ProverConfig, proof_output_dir: Option, @@ -228,7 +225,7 @@ pub async fn prove( // Prove the block let block_proof = if prover_config.test_only { block - .prove_test(segment_proof_runtime, previous_block_proof, prover_config) + .prove_test(proof_runtime, previous_block_proof, prover_config) .then(move |proof| async move { let proof = proof?; let block_number = proof.b_height; @@ -252,12 +249,7 @@ pub async fn prove( .await? } else { block - .prove( - block_proof_runtime, - segment_proof_runtime, - previous_block_proof, - prover_config, - ) + .prove(proof_runtime, previous_block_proof, prover_config) .then(move |proof| async move { let proof = proof?; let block_number = proof.b_height; diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 0409e399b..941521a3c 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -102,7 +102,7 @@ fi if [[ $8 == "test_only" ]]; then # test only run echo "Proving blocks ${BLOCK_INTERVAL} in a test_only mode now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --runtime in-memory --test-only --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command retVal=$? @@ -125,7 +125,7 @@ if [[ $8 == "test_only" ]]; then else # normal run echo "Proving blocks ${BLOCK_INTERVAL} now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES"' + command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL --proof-output-dir $PROOF_OUTPUT_DIR $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command echo -e "Proof generation finished with result: $?" From 3701b1e5110ac2fa7e4480209fb6d83e2ac3d10d Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 10 Sep 2024 19:41:50 +0530 Subject: [PATCH 7/9] testing changes --- Cargo.lock | 451 ++++++++++++++++++++---------------- zero_bin/prover/src/lib.rs | 34 +-- zero_bin/tools/prove_rpc.sh | 4 +- 3 files changed, 268 insertions(+), 221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf76f812c..368326390 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aes" @@ -58,14 +58,15 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eadd758805fe353ea8a520da4531efb9739382312c354d3e90b7a16353b0315" +checksum = "b87410cf02aeaeca18efe1f807893509ed716be9d1514ae1e6aadc60660a49a6" dependencies = [ "alloy-consensus", "alloy-core", "alloy-eips", "alloy-json-rpc", + "alloy-network", "alloy-provider", "alloy-rpc-client", "alloy-rpc-types", @@ -75,9 +76,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb07629a5d0645d29f68d2fb6f4d0cf15c89ec0965be915f303967180929743f" +checksum = "2b4f201b0ac8f81315fbdc55269965a8ddadbc04ab47fa65a1a468f9a40f7a5f" dependencies = [ "num_enum", "strum", @@ -85,9 +86,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7198a527b4c4762cb88d54bcaeb0428f4298b72552c9c8ec4af614b4a4990c59" +checksum = "40a70d19a83dfee0cd4b16d28d2fc1c822a9a55935c672259dd8165e342c4147" dependencies = [ "alloy-eips", "alloy-primitives", @@ -99,9 +100,9 @@ dependencies = [ [[package]] name = "alloy-core" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6dbb79f4e3285cc87f50c0d4be9a3a812643623b2e3558d425b41cbd795ceb" +checksum = "88b095eb0533144b4497e84a9cc3e44a5c2e3754a3983c0376a55a2f9183a53e" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -131,9 +132,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159eab0e4e15b88571f55673af37314f4b8f17630dc1b393c3d70f2128a1d494" +checksum = "9ecd43a1ce87109f4d64efeff2b6fd0d7ff06afe93c4ffeb4e23bbb34d77ce84" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -141,6 +142,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "c-kzg", + "derive_more", "once_cell", "serde", "sha2", @@ -148,9 +150,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7733446dd531f8eb877331fea02f6c40bdbb47444a17dc3464bf75319cc073a" +checksum = "92fdc402c955dc61b415325c1ff36988bd1d2287146b8123f0f997013343a118" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -162,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b80851d1697fc4fa2827998e3ee010a3d1fc59c7d25e87070840169fcf465832" +checksum = "7da63445b84f4cd2c5b5b62b7ebf57125e4b9ce233d45db4d2b4a0f329f5fe5a" dependencies = [ "alloy-consensus", "alloy-eips", @@ -183,10 +185,11 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76a2336889f3d0624b18213239d27f4f34eb476eb35bef22f6a8cc24e0c0078" +checksum = "97fb528a622b49b51c79d4ba61e3ff2567b31bd339571e8e06e36b94494cc49a" dependencies = [ + "alloy-eips", "alloy-primitives", "alloy-serde", "serde", @@ -194,9 +197,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a767e59c86900dd7c3ce3ecef04f3ace5ac9631ee150beb8b7d22f7fa3bbb2d7" +checksum = "411aff151f2a73124ee473708e82ed51b2535f68928b6a1caa8bc1246ae6f7cd" dependencies = [ "alloy-rlp", "bytes", @@ -216,9 +219,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d2a195caa6707f5ce13905794865765afc6d9ea92c3a56e3a973c168d703bc" +checksum = "9b624bfbfa145b64571650ce951eba3ec1d38b781e4fe2b5a8e7410f99f73336" dependencies = [ "alloy-chains", "alloy-consensus", @@ -235,7 +238,7 @@ dependencies = [ "async-stream", "async-trait", "auto_impl", - "dashmap 6.0.1", + "dashmap 6.1.0", "futures", "futures-utils-wasm", "lru", @@ -268,14 +271,14 @@ checksum = "4d0f2d905ebd295e7effec65e5f6868d153936130ae718352771de3e7d03c75c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "alloy-rpc-client" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed31cdba2b23d71c555505b06674f8e7459496abfd7f4875d268434ef5a99ee6" +checksum = "02af2846b186e4ae25eb448dd78aebdb82f55668bdc360a3a7bf4c7754cb8251" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -287,16 +290,16 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tower", + "tower 0.5.1", "tracing", "url", ] [[package]] name = "alloy-rpc-types" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d758f65aa648491c6358335c578de45cd7de6fdf2877c3cef61f2c9bebea21" +checksum = "6f1407e7da4a25fef7d4e6214e887b0b084e8ff02e2c26506ad55c0aa972ca10" dependencies = [ "alloy-rpc-types-eth", "alloy-rpc-types-trace", @@ -306,9 +309,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ba05d6ee4db0d89113294a614137940f79abfc2c40a9a3bee2995660358776" +checksum = "48cc832ec2ad270ac9859cc5056117890af943686d432a2c478eafef5102e8eb" dependencies = [ "alloy-consensus", "alloy-eips", @@ -325,9 +328,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd2af822ed58f2b6dd7cfccf88bf69f42c9a8cbf4663316227646a8a3e5a591f" +checksum = "27b750a686a279a7d1722dc2089cace14d1de90b6317caa49c9369ad8cf7c5bd" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -339,9 +342,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd260ede54f0b53761fdd04133acc10ae70427f66a69aa9590529bbd066cd58" +checksum = "da1c4dbeff85bba7bbacb5df2a05e68e426aed994669785017a9d33b4cee32f4" dependencies = [ "alloy-primitives", "serde", @@ -350,9 +353,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5193ee6b370b89db154d7dc40c6a8e6ce11213865baaf2b418a9f2006be762" +checksum = "d458fc2eec44beb3c64cb987941e992177390ecd47b291eeb3d1ec264b5f70ff" dependencies = [ "alloy-primitives", "async-trait", @@ -364,56 +367,56 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183bcfc0f3291d9c41a3774172ee582fb2ce6eb6569085471d8f225de7bb86fc" +checksum = "0458ccb02a564228fcd76efb8eb5a520521a8347becde37b402afec9a1b83859" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "alloy-sol-macro-expander" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c4d842beb7a6686d04125603bc57614d5ed78bf95e4753274db3db4ba95214" +checksum = "2bc65475025fc1e84bf86fc840f04f63fcccdcf3cf12053c99918e4054dfbc69" dependencies = [ "alloy-sol-macro-input", "const-hex", - "heck 0.5.0", + "heck", "indexmap", - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "syn-solidity", "tiny-keccak", ] [[package]] name = "alloy-sol-macro-input" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1306e8d3c9e6e6ecf7a39ffaf7291e73a5f655a2defd366ee92c2efebcdf7fee" +checksum = "6ed10f0715a0b69fde3236ff3b9ae5f6f7c97db5a387747100070d3016b9266b" dependencies = [ "const-hex", "dunce", - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "syn-solidity", ] [[package]] name = "alloy-sol-types" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "577e262966e92112edbd15b1b2c0947cc434d6e8311df96d3329793fe8047da9" +checksum = "1eb88e4da0a1b697ed6a9f811fdba223cf4d5c21410804fd1707836af73a462b" dependencies = [ "alloy-primitives", "alloy-sol-macro", @@ -422,9 +425,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "454220c714857cf68af87d788d1f0638ad8766268b94f6a49fed96cbc2ab382c" +checksum = "a98b7bf2a9e0671a1e914e89854e74c89e62a549da548184c6d93c2be828a575" dependencies = [ "alloy-json-rpc", "base64", @@ -434,22 +437,22 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tower", + "tower 0.5.1", "tracing", "url", ] [[package]] name = "alloy-transport-http" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377f2353d7fea03a2dba6b9ffbb7d610402c040dd5700d1fae8b9ec2673eed9b" +checksum = "85d928f36065b0f548782ec2acf4b54a5cf8b297a22ca18d2c8fe165af90561f" dependencies = [ "alloy-json-rpc", "alloy-transport", "reqwest", "serde_json", - "tower", + "tower 0.5.1", "tracing", "url", ] @@ -559,9 +562,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" dependencies = [ "backtrace", ] @@ -720,7 +723,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "synstructure", ] @@ -732,7 +735,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -749,9 +752,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" dependencies = [ "async-task", "concurrent-queue", @@ -819,7 +822,7 @@ dependencies = [ "futures-lite 2.3.0", "parking", "polling 3.7.3", - "rustix 0.38.35", + "rustix 0.38.36", "slab", "tracing", "windows-sys 0.59.0", @@ -876,7 +879,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -887,13 +890,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -919,7 +922,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -956,7 +959,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -999,17 +1002,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -1203,9 +1206,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.15" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "shlex", ] @@ -1255,9 +1258,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -1265,9 +1268,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -1281,10 +1284,10 @@ version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1379,12 +1382,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "cookie-factory" version = "0.3.3" @@ -1393,9 +1390,9 @@ checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" [[package]] name = "copyvec" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540a4ee3e6ea3547a6b492c5e76a4b9086e6b41178061e03ca82fc385912b1f" +checksum = "8aba112395a3627b61476b950f4c015964642a1d32a44112ad97c4a781dab81f" [[package]] name = "core-foundation" @@ -1415,9 +1412,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1579,7 +1576,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1590,7 +1587,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1608,9 +1605,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1661,7 +1658,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1686,46 +1683,54 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "derive_builder_macro" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc" dependencies = [ "derive_builder_core", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "derive_more" -version = "0.99.18" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ - "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.1", - "syn 2.0.76", + "syn 2.0.77", + "unicode-xid", ] [[package]] @@ -1787,7 +1792,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1861,14 +1866,14 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2234,7 +2239,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2317,9 +2322,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -2383,12 +2388,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -2521,9 +2520,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -2534,7 +2533,7 @@ dependencies = [ "pin-project-lite", "socket2 0.5.7", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] @@ -2606,9 +2605,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", @@ -2655,9 +2654,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "is-terminal" @@ -2876,7 +2875,7 @@ checksum = "cb26336e6dc7cc76e7927d2c9e7e3bb376d7af65a6f56a0b16c47d18a9b1abc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2952,11 +2951,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -3145,7 +3144,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3168,9 +3167,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] @@ -3219,7 +3218,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3291,7 +3290,7 @@ dependencies = [ [[package]] name = "paladin-core" version = "0.4.2" -source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#6c752fd2ec9bc81dc2f4b727580d7dd7a42b9165" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#b9e864487a2250e9f0d1277e34f6be4d26fd2241" dependencies = [ "anyhow", "async-trait", @@ -3323,10 +3322,10 @@ dependencies = [ [[package]] name = "paladin-opkind-derive" version = "0.4.1" -source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#6c752fd2ec9bc81dc2f4b727580d7dd7a42b9165" +source = "git+https://github.com/0xPolygonZero/paladin.git?branch=arpit/507-2#b9e864487a2250e9f0d1277e34f6be4d26fd2241" dependencies = [ "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3357,9 +3356,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -3417,9 +3416,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea" dependencies = [ "memchr", "thiserror", @@ -3428,9 +3427,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "664d22978e2815783adbdd2c588b455b1bd625299ce36b2a99881ac9627e6d8d" dependencies = [ "pest", "pest_generator", @@ -3438,22 +3437,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "a2d5487022d5d33f4c30d91c22afa240ce2a644e87fe08caad974d4eab6badbe" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "0091754bbd0ea592c4deb3a122ce8ecbb0753b738aa82bc055fcc2eccc8d8174" dependencies = [ "once_cell", "pest", @@ -3477,7 +3476,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3624,9 +3623,9 @@ source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e1 [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -3637,15 +3636,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -3676,7 +3675,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.35", + "rustix 0.38.36", "tracing", "windows-sys 0.59.0", ] @@ -3775,6 +3774,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -4123,7 +4144,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tower", + "tower 0.4.13", "trace_decoder", "tracing", "tracing-subscriber", @@ -4158,7 +4179,7 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.76", + "syn 2.0.77", "unicode-ident", ] @@ -4247,9 +4268,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags 2.6.0", "errno", @@ -4316,9 +4337,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -4369,11 +4390,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4459,9 +4480,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -4477,20 +4498,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -4715,11 +4736,11 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4741,9 +4762,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -4752,14 +4773,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284c41c2919303438fcf8dede4036fd1e82d4fc0fbb2b279bd2a1442c909ca92" +checksum = "4b95156f8b577cb59dc0b1df15c6f29a10afc5f8a7ac9786b0b5c68c19149278" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4785,7 +4806,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4815,7 +4836,7 @@ dependencies = [ "cfg-if", "fastrand 2.1.1", "once_cell", - "rustix 0.38.35", + "rustix 0.38.36", "windows-sys 0.59.0", ] @@ -4845,7 +4866,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4936,9 +4957,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -4971,7 +4992,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -5000,9 +5021,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -5012,9 +5033,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -5073,6 +5094,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -5147,7 +5182,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -5217,9 +5252,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "u4" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068a8b5939889cb4b24fd99201f2c7bc142a25986ae0eb903cb05537484192d5" +checksum = "390884f06a6f4eee34a2082a3eb17844cf3605a69f033f79bf0e6f460279112c" dependencies = [ "const-default", "serde", @@ -5270,6 +5305,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-xid" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" + [[package]] name = "unroll" version = "0.1.5" @@ -5426,7 +5467,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -5460,7 +5501,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5807,7 +5848,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -5827,7 +5868,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -5846,6 +5887,6 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "trybuild", ] diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index a5600db6a..97a80df6a 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -6,8 +6,14 @@ use std::sync::Arc; use alloy::primitives::U256; use anyhow::{Context, Result}; -use futures::{future::BoxFuture, FutureExt, TryFutureExt, TryStreamExt}; +use evm_arithmetization::SegmentDataIterator; +use futures::StreamExt; +use futures::{ + future, future::BoxFuture, stream::FuturesOrdered, stream::FuturesUnordered, FutureExt, + TryFutureExt, TryStreamExt, +}; use num_traits::ToPrimitive as _; +use paladin::directive::{Directive, IndexedStream}; use proof_gen::proof_types::GeneratedBlockProof; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; @@ -47,9 +53,6 @@ impl BlockProverInput { prover_config: Arc, ) -> Result { use anyhow::Context as _; - use evm_arithmetization::SegmentDataIterator; - use futures::{stream::FuturesUnordered, FutureExt}; - use paladin::directive::{Directive, IndexedStream}; let ProverConfig { max_cpu_len_log, @@ -83,26 +86,30 @@ impl BlockProverInput { save_inputs_on_error, }; - // Segment the batches, prove segments and aggregate them to resulting batch - // proofs. - let batch_proof_futs: FuturesUnordered<_> = block_generation_inputs + // Segment the batches, prove segments, and aggregate the segment proofs. + let vec_segment_agg_proofs: FuturesOrdered<_> = block_generation_inputs .iter() .enumerate() - .map(|(idx, txn_batch)| { + .map(|(_, txn_batch)| { let segment_data_iterator = SegmentDataIterator::::new( txn_batch, Some(max_cpu_len_log), ); + // Map over the indexed stream with the given seg_prove_ops operation. Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) - .fold(&seg_agg_ops) .run(&proof_runtime.segment_proof_runtime) - .map(move |e| { - e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) - }) }) .collect(); + // Collect results of segment proofs and prepare for batch aggregation + let batch_proof_futs = vec_segment_agg_proofs.map(|segment_proofs| { + let x = segment_proofs.unwrap(); + Directive::fold(IndexedStream::new(x), &seg_agg_ops) + .run(&proof_runtime.block_proof_runtime) + .map(move |e| e.map(|p| (proof_gen::proof_types::BatchAggregatableProof::from(p)))) + }); + // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) @@ -142,7 +149,6 @@ impl BlockProverInput { ) -> Result { use std::iter::repeat; - use futures::future; use paladin::directive::{Directive, IndexedStream}; let ProverConfig { @@ -177,7 +183,7 @@ impl BlockProverInput { ); simulation - .run(&proof_runtime.segment_proof_runtime) + .run(&proof_runtime.block_proof_runtime) .await? .try_for_each(|_| future::ok(())) .await?; diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 1c7420491..0c5e19c96 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -110,7 +110,7 @@ fi if [[ $8 == "test_only" ]]; then # test only run echo "Proving blocks ${BLOCK_INTERVAL} in a test_only mode now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --test-only --worker-run-mode split --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command retVal=$? @@ -133,7 +133,7 @@ if [[ $8 == "test_only" ]]; then else # normal run echo "Proving blocks ${BLOCK_INTERVAL} now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --worker-run-mode split --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command echo -e "Proof generation finished with result: $?" From c410daaff205b0f12f7a628f37df1daeb5eb82fb Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 10 Sep 2024 21:49:50 +0530 Subject: [PATCH 8/9] separate segment proof and segment agg proof --- zero_bin/prover/src/lib.rs | 32 +++++++++++++++++++------------- zero_bin/tools/prove_rpc.sh | 4 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index 97a80df6a..cb0fe13ba 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -9,8 +9,7 @@ use anyhow::{Context, Result}; use evm_arithmetization::SegmentDataIterator; use futures::StreamExt; use futures::{ - future, future::BoxFuture, stream::FuturesOrdered, stream::FuturesUnordered, FutureExt, - TryFutureExt, TryStreamExt, + future, future::BoxFuture, stream::FuturesUnordered, FutureExt, TryFutureExt, TryStreamExt, }; use num_traits::ToPrimitive as _; use paladin::directive::{Directive, IndexedStream}; @@ -86,11 +85,9 @@ impl BlockProverInput { save_inputs_on_error, }; - // Segment the batches, prove segments, and aggregate the segment proofs. - let vec_segment_agg_proofs: FuturesOrdered<_> = block_generation_inputs + let vec_segment_agg_proofs: FuturesUnordered<_> = block_generation_inputs .iter() - .enumerate() - .map(|(_, txn_batch)| { + .map(|txn_batch| { let segment_data_iterator = SegmentDataIterator::::new( txn_batch, Some(max_cpu_len_log), @@ -102,13 +99,22 @@ impl BlockProverInput { }) .collect(); - // Collect results of segment proofs and prepare for batch aggregation - let batch_proof_futs = vec_segment_agg_proofs.map(|segment_proofs| { - let x = segment_proofs.unwrap(); - Directive::fold(IndexedStream::new(x), &seg_agg_ops) - .run(&proof_runtime.block_proof_runtime) - .map(move |e| e.map(|p| (proof_gen::proof_types::BatchAggregatableProof::from(p)))) - }); + // Await all the futures concurrently + let vec_segment_agg_proofs: Vec<_> = vec_segment_agg_proofs.collect().await; + + let batch_proof_futs: FuturesUnordered<_> = vec_segment_agg_proofs + .into_iter() + .enumerate() + .map(|(idx, result)| { + // todo - check how to handle the unwrap + let res = result.unwrap(); + Directive::fold(IndexedStream::new(res), &seg_agg_ops) + .run(&proof_runtime.block_proof_runtime) + .map(move |e| { + e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) + }) + }) + .collect(); // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 0c5e19c96..1c7420491 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -110,7 +110,7 @@ fi if [[ $8 == "test_only" ]]; then # test only run echo "Proving blocks ${BLOCK_INTERVAL} in a test_only mode now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --test-only --worker-run-mode split --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command retVal=$? @@ -133,7 +133,7 @@ if [[ $8 == "test_only" ]]; then else # normal run echo "Proving blocks ${BLOCK_INTERVAL} now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --bin leader -- --worker-run-mode split --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + command='cargo r --release --bin leader -- --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' if [ "$OUTPUT_TO_TERMINAL" = true ]; then eval $command echo -e "Proof generation finished with result: $?" From fab79354a8307341ed2c80bfb72dd50033c64005 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Fri, 13 Sep 2024 18:53:28 +0530 Subject: [PATCH 9/9] review comments --- zero_bin/common/src/proof_runtime.rs | 4 ++-- zero_bin/leader/src/cli.rs | 8 +++---- zero_bin/leader/src/client.rs | 4 ++-- zero_bin/leader/src/main.rs | 35 ++++++++++++++-------------- zero_bin/leader/src/stdio.rs | 4 ++-- zero_bin/prover/src/lib.rs | 32 ++++++++----------------- 6 files changed, 37 insertions(+), 50 deletions(-) diff --git a/zero_bin/common/src/proof_runtime.rs b/zero_bin/common/src/proof_runtime.rs index b343b0ae9..849a4fa52 100644 --- a/zero_bin/common/src/proof_runtime.rs +++ b/zero_bin/common/src/proof_runtime.rs @@ -1,6 +1,6 @@ use paladin::runtime::Runtime; pub struct ProofRuntime { - pub block_proof_runtime: Runtime, - pub segment_proof_runtime: Runtime, + pub light_proof_runtime: Runtime, + pub heavy_proof_runtime: Runtime, } diff --git a/zero_bin/leader/src/cli.rs b/zero_bin/leader/src/cli.rs index 1c7067ca4..a44b074ef 100644 --- a/zero_bin/leader/src/cli.rs +++ b/zero_bin/leader/src/cli.rs @@ -25,15 +25,15 @@ pub(crate) struct Cli { #[clap(flatten)] pub(crate) prover_state_config: CliProverStateConfig, - // Mode to use for worker for setup (split or unified) - #[arg(long = "worker-run-mode", help_heading = WORKER_HELP_HEADING, value_enum, default_value = "unified")] + // Mode to use for worker for setup (affinity or default) + #[arg(long = "worker-run-mode", help_heading = WORKER_HELP_HEADING, value_enum, default_value = "default")] pub(crate) worker_run_mode: WorkerRunMode, } #[derive(ValueEnum, Clone, PartialEq, Debug)] pub enum WorkerRunMode { - Split, - Unified, + Affinity, + Default, } #[derive(Subcommand)] diff --git a/zero_bin/leader/src/client.rs b/zero_bin/leader/src/client.rs index 36002b1e6..5d5f42da6 100644 --- a/zero_bin/leader/src/client.rs +++ b/zero_bin/leader/src/client.rs @@ -116,8 +116,8 @@ pub(crate) async fn client_main( } } - proof_runtime.block_proof_runtime.close().await?; - proof_runtime.segment_proof_runtime.close().await?; + proof_runtime.light_proof_runtime.close().await?; + proof_runtime.heavy_proof_runtime.close().await?; if test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index 79eb8976f..d6a943fa6 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -38,8 +38,8 @@ fn get_previous_proof(path: Option) -> Result Result<()> { let args = cli::Cli::parse(); - let mut block_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); - let mut segment_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); - if args.worker_run_mode == cli::WorkerRunMode::Split { - // If we're running in split mode, we need to set the routing key for the + let mut light_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + let mut heavy_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + if args.worker_run_mode == cli::WorkerRunMode::Affinity { + // If we're running in affinity mode, we need to set the routing key for the // block proof and segment proof. - info!("Workers running in split mode"); - block_proof_routing_key = BLOCK_PROOF_ROUTING_KEY.to_string(); - segment_proof_routing_key = SEGMENT_PROOF_ROUTING_KEY.to_string(); + info!("Workers running in affinity mode"); + light_proof_routing_key = LIGHT_PROOF_ROUTING_KEY.to_string(); + heavy_proof_routing_key = HEAVY_PROOF_ROUTING_KEY.to_string(); } - let mut block_proof_paladin_args = args.paladin.clone(); - block_proof_paladin_args.task_bus_routing_key = Some(block_proof_routing_key); + let mut light_proof_paladin_args = args.paladin.clone(); + light_proof_paladin_args.task_bus_routing_key = Some(light_proof_routing_key); - let mut segment_proof_paladin_args = args.paladin.clone(); - segment_proof_paladin_args.task_bus_routing_key = Some(segment_proof_routing_key); + let mut heavy_proof_paladin_args = args.paladin.clone(); + heavy_proof_paladin_args.task_bus_routing_key = Some(heavy_proof_routing_key); - let block_proof_runtime = Runtime::from_config(&block_proof_paladin_args, register()).await?; - let segment_proof_runtime = - Runtime::from_config(&segment_proof_paladin_args, register()).await?; + let light_proof_runtime = Runtime::from_config(&light_proof_paladin_args, register()).await?; + let heavy_proof_runtime = Runtime::from_config(&heavy_proof_paladin_args, register()).await?; if let Command::Clean = args.command { return zero_bin_common::prover_state::persistence::delete_all(); } let proof_runtime = ProofRuntime { - block_proof_runtime, - segment_proof_runtime, + light_proof_runtime, + heavy_proof_runtime, }; let proof_runtime = Arc::new(proof_runtime); diff --git a/zero_bin/leader/src/stdio.rs b/zero_bin/leader/src/stdio.rs index 71d59eca2..5344184ae 100644 --- a/zero_bin/leader/src/stdio.rs +++ b/zero_bin/leader/src/stdio.rs @@ -54,8 +54,8 @@ pub(crate) async fn stdio_main( } } - proof_runtime.block_proof_runtime.close().await?; - proof_runtime.segment_proof_runtime.close().await?; + proof_runtime.light_proof_runtime.close().await?; + proof_runtime.heavy_proof_runtime.close().await?; if prover_config.test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero_bin/prover/src/lib.rs b/zero_bin/prover/src/lib.rs index d3c550bd5..3968b9f06 100644 --- a/zero_bin/prover/src/lib.rs +++ b/zero_bin/prover/src/lib.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use alloy::primitives::U256; use anyhow::{Context, Result}; use evm_arithmetization::SegmentDataIterator; -use futures::StreamExt; use futures::{ future, future::BoxFuture, stream::FuturesUnordered, FutureExt, TryFutureExt, TryStreamExt, }; @@ -98,31 +97,20 @@ impl BlockProverInput { save_inputs_on_error, }; - let vec_segment_agg_proofs: FuturesUnordered<_> = block_generation_inputs + // Segment the batches, prove segments and aggregate them to resulting batch + // proofs. + let batch_proof_futs: FuturesUnordered<_> = block_generation_inputs .iter() - .map(|txn_batch| { + .enumerate() + .map(|(idx, txn_batch)| { let segment_data_iterator = SegmentDataIterator::::new( txn_batch, Some(max_cpu_len_log), ); - // Map over the indexed stream with the given seg_prove_ops operation. Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) - .run(&proof_runtime.segment_proof_runtime) - }) - .collect(); - - // Await all the futures concurrently - let vec_segment_agg_proofs: Vec<_> = vec_segment_agg_proofs.collect().await; - - let batch_proof_futs: FuturesUnordered<_> = vec_segment_agg_proofs - .into_iter() - .enumerate() - .map(|(idx, result)| { - // todo - check how to handle the unwrap - let res = result.unwrap(); - Directive::fold(IndexedStream::new(res), &seg_agg_ops) - .run(&proof_runtime.block_proof_runtime) + .fold(&seg_agg_ops) + .run(&proof_runtime.heavy_proof_runtime) .map(move |e| { e.map(|p| (idx, proof_gen::proof_types::BatchAggregatableProof::from(p))) }) @@ -132,7 +120,7 @@ impl BlockProverInput { // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) - .run(&proof_runtime.block_proof_runtime) + .run(&proof_runtime.light_proof_runtime) .await?; if let proof_gen::proof_types::BatchAggregatableProof::Agg(proof) = final_batch_proof { @@ -149,7 +137,7 @@ impl BlockProverInput { prev, save_inputs_on_error, }) - .run(&proof_runtime.block_proof_runtime) + .run(&proof_runtime.light_proof_runtime) .await?; info!("Successfully proved block {block_number}"); @@ -202,7 +190,7 @@ impl BlockProverInput { ); simulation - .run(&proof_runtime.block_proof_runtime) + .run(&proof_runtime.light_proof_runtime) .await? .try_for_each(|_| future::ok(())) .await?;