From 7b01f5d65be4087c76110a345c6d28a8548c8389 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 15 Jul 2024 16:49:51 +0800 Subject: [PATCH 01/53] Implement JumpDest fetching from RPC. Supports CREATE1/CREATE2 Supports transaction deployments Includes test scripts for randomised testing --- .gitignore | 1 + Cargo.lock | 4 + Cargo.toml | 4 +- evm_arithmetization/Cargo.toml | 1 + .../benches/fibonacci_25m_gas.rs | 1 + .../src/cpu/kernel/interpreter.rs | 60 ++- .../src/cpu/kernel/tests/add11.rs | 2 + .../kernel/tests/core/jumpdest_analysis.rs | 24 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 1 + .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/jumpdest.rs | 203 +++++++++ evm_arithmetization/src/generation/mod.rs | 11 + .../src/generation/prover_input.rs | 48 +- evm_arithmetization/src/generation/state.rs | 3 +- evm_arithmetization/src/lib.rs | 3 + evm_arithmetization/tests/add11_yml.rs | 1 + evm_arithmetization/tests/erc20.rs | 1 + evm_arithmetization/tests/erc721.rs | 1 + evm_arithmetization/tests/global_exit_root.rs | 1 + evm_arithmetization/tests/log_opcode.rs | 1 + evm_arithmetization/tests/selfdestruct.rs | 1 + evm_arithmetization/tests/simple_transfer.rs | 1 + evm_arithmetization/tests/withdrawals.rs | 1 + test_jerigon.sh | 88 ++++ test_native.sh | 83 ++++ trace_decoder/src/core.rs | 12 + trace_decoder/src/interface.rs | 20 +- trace_decoder/src/jumpdest.rs | 1 + trace_decoder/src/lib.rs | 1 + zero_bin/prover/src/cli.rs | 2 +- zero_bin/rpc/Cargo.toml | 3 + zero_bin/rpc/src/jerigon.rs | 134 +++++- zero_bin/rpc/src/jumpdest.rs | 417 ++++++++++++++++++ zero_bin/rpc/src/lib.rs | 1 + zero_bin/rpc/src/main.rs | 3 + zero_bin/rpc/src/native/mod.rs | 6 +- zero_bin/rpc/src/native/txn.rs | 72 ++- zero_bin/tools/prove_stdio.sh | 22 +- 38 files changed, 1170 insertions(+), 71 deletions(-) create mode 100644 evm_arithmetization/src/generation/jumpdest.rs create mode 100755 test_jerigon.sh create mode 100755 test_native.sh create mode 100644 trace_decoder/src/jumpdest.rs create mode 100644 zero_bin/rpc/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index f035f7002..dd5a27331 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .vscode /**/*.ignoreme **/output.log +witnesses diff --git a/Cargo.lock b/Cargo.lock index 1791c16ed..0b4835ef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2066,6 +2066,7 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", + "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", @@ -4169,6 +4170,8 @@ version = "0.1.0" dependencies = [ "alloy", "alloy-compat", + "alloy-primitives", + "alloy-serde", "anyhow", "cargo_metadata", "clap", @@ -4177,6 +4180,7 @@ dependencies = [ "futures", "hex", "itertools 0.13.0", + "keccak-hash 0.10.0", "mpt_trie", "primitive-types 0.12.2", "proof_gen", diff --git a/Cargo.toml b/Cargo.toml index 9628a345c..84a438806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ alloy = { version = '0.3.0', default-features = false, features = [ "transport-http", "rpc-types-debug", ] } +alloy-primitives = "0.8.0" +alloy-serde = "0.3.0" anyhow = "1.0.86" async-stream = "0.3.5" axum = "0.7.5" @@ -57,7 +59,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -94,7 +95,6 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" -serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index ffdb8d2f5..e8e832c2d 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,6 +15,7 @@ homepage.workspace = true keywords.workspace = true [dependencies] +__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index f6d5fc39f..57b902c63 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,6 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 10af8d495..d0cb867fa 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -5,10 +5,12 @@ //! the future execution and generate nondeterministically the corresponding //! jumpdest table, before the actual CPU carries on with contract execution. +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; +use keccak_hash::H256; use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; @@ -19,7 +21,9 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; +use crate::generation::jumpdest::{ContextJumpDests, JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; +use crate::generation::prover_input::get_proofs_and_jumpdests; use crate::generation::rlp::all_rlp_prover_inputs_reversed; use crate::generation::state::{ all_ger_prover_inputs, all_withdrawals_prover_inputs_reversed, GenerationState, @@ -56,6 +60,7 @@ pub(crate) struct Interpreter { /// Counts the number of appearances of each opcode. For debugging purposes. #[allow(unused)] pub(crate) opcode_count: [usize; 0x100], + /// A table of contexts and their reached JUMPDESTs. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -71,9 +76,9 @@ pub(crate) struct Interpreter { pub(crate) fn simulate_cpu_and_get_user_jumps( final_label: &str, state: &GenerationState, -) -> Option>> { +) -> Option<(JumpDestTableProcessed, JumpDestTableWitness)> { match state.jumpdest_table { - Some(_) => None, + Some(_) => Default::default(), None => { let halt_pc = KERNEL.global_labels[final_label]; let initial_context = state.registers.context; @@ -92,16 +97,15 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - interpreter + let jdtw = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table); + .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - - interpreter.generation_state.jumpdest_table + (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) } } } @@ -114,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, - pub(crate) jumpdest_table: Option>>, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, + pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } @@ -150,6 +154,48 @@ pub(crate) fn set_registers_and_run( interpreter.run() } +/// Computes the JUMPDEST proofs for each context. +/// +/// # Arguments +/// +/// - `jumpdest_table_rpc`: The raw table received from RPC. +/// - `code_db`: The corresponding database of contract code used in the trace. +pub(crate) fn set_jumpdest_analysis_inputs_rpc( + jumpdest_table_rpc: &JumpDestTableWitness, + code_map: &HashMap>, +) -> JumpDestTableProcessed { + let ctx_proofs = (*jumpdest_table_rpc) + .iter() + .flat_map(|(code_addr, ctx_jumpdests)| { + prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + }) + .collect(); + JumpDestTableProcessed::new(ctx_proofs) +} + +/// Orchestrates the proving of all contexts in a specific bytecode. +/// +/// # Arguments +/// +/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached +/// `JUMPDEST`s. +/// - `code`: The bytecode for the contexts. This is the same for all contexts. +fn prove_context_jumpdests( + code: &[u8], + ctx_jumpdests: &ContextJumpDests, +) -> HashMap> { + ctx_jumpdests + .0 + .iter() + .map(|(&ctx, jumpdests)| { + let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { + get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) + }); + (ctx, proofs) + }) + .collect() +} + impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fe36b40e3..1cc5588e1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,6 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; @@ -371,6 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index b0ef17033..1d93e0e31 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -7,6 +7,7 @@ use plonky2::field::goldilocks_field::GoldilocksField as F; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::interpreter::Interpreter; use crate::cpu::kernel::opcodes::{get_opcode, get_push_opcode}; +use crate::generation::jumpdest::JumpDestTableProcessed; use crate::witness::operation::CONTEXT_SCALING_FACTOR; #[test] @@ -67,7 +68,10 @@ fn test_jumpdest_analysis() -> Result<()> { interpreter.generation_state.jumpdest_table, // Context 3 has jumpdest 1, 5, 7. All have proof 0 and hence // the list [proof_0, jumpdest_0, ... ] is [0, 1, 0, 5, 0, 7, 8, 40] - Some(HashMap::from([(3, vec![0, 1, 0, 5, 0, 7, 8, 40])])) + Some(JumpDestTableProcessed::new(HashMap::from([( + 3, + vec![0, 1, 0, 5, 0, 7, 8, 40] + )]))) ); // Run jumpdest analysis with context = 3 @@ -84,14 +88,14 @@ fn test_jumpdest_analysis() -> Result<()> { // We need to manually pop the jumpdest_table and push its value on the top of // the stack - interpreter + (*interpreter .generation_state .jumpdest_table .as_mut() - .unwrap() - .get_mut(&CONTEXT) - .unwrap() - .pop(); + .unwrap()) + .get_mut(&CONTEXT) + .unwrap() + .pop(); interpreter .push(41.into()) .expect("The stack should not overflow"); @@ -136,7 +140,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from( + [(3, vec![1, 33])], + ))); interpreter.run()?; @@ -149,7 +155,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new( + HashMap::from([(3, vec![1, 33])]), + )); assert!(interpreter.run().is_err()); diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index be88021de..08fe57b1a 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,6 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, + jumpdest_tables: vec![], }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index 39810148b..08b17d495 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -249,7 +249,7 @@ impl Interpreter { } pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - self.generation_state.set_jumpdest_analysis_inputs(jumps); + let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); } pub(crate) fn extract_kernel_memory(self, segment: Segment, range: Range) -> Vec { diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs new file mode 100644 index 000000000..fc2a8e1a6 --- /dev/null +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -0,0 +1,203 @@ +use std::cmp::max; +use std::{ + collections::{BTreeSet, HashMap}, + fmt::Display, + ops::{Deref, DerefMut}, +}; + +use itertools::{sorted, Itertools}; +use keccak_hash::H256; +use serde::{Deserialize, Serialize}; + +/// Each `CodeAddress` can be called one or more times, each time creating a new +/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct ContextJumpDests(pub HashMap>); + +/// The result after proving a `JumpDestTableWitness`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct JumpDestTableProcessed(HashMap>); + +/// Map `CodeAddress -> (Context -> [JumpDests])` +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableWitness(HashMap); + +impl ContextJumpDests { + pub fn insert(&mut self, ctx: usize, offset: usize) { + self.entry(ctx).or_default().insert(offset); + } + + pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { + self.0.get(&ctx) + } +} + +impl JumpDestTableProcessed { + pub fn new(ctx_map: HashMap>) -> Self { + Self(ctx_map) + } +} + +impl JumpDestTableWitness { + pub fn new(ctx_map: HashMap) -> Self { + Self(ctx_map) + } + + pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { + self.0.get(code_hash) + } + + /// Insert `offset` into `ctx` under the corresponding `code_hash`. + /// Creates the required `ctx` keys and `code_hash`. Idempotent. + pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { + (*self).entry(code_hash).or_default().insert(ctx, offset); + + // TODO(einar) remove before publishing PR. + assert!(self.0.contains_key(&code_hash)); + assert!(self.0[&code_hash].0.contains_key(&ctx)); + assert!(self.0[&code_hash].0[&ctx].contains(&offset)); + } + + pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { + let mut curr_max_ctx = prev_max_ctx; + + // TODO: Opportunity for optimization: Simulate to generate only missing + // JUMPDEST tables. + for (code_hash, ctx_tbl) in (*other).iter() { + for (ctx, jumpdests) in ctx_tbl.0.iter() { + let batch_ctx = prev_max_ctx + ctx; + curr_max_ctx = max(curr_max_ctx, batch_ctx); + + for offset in jumpdests { + self.insert(*code_hash, batch_ctx, *offset); + + assert!(self.0.contains_key(code_hash)); + assert!(self.0[code_hash].0.contains_key(&batch_ctx)); + assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); + } + // dbg!(&self); + } + } + + (self, curr_max_ctx) + } + + pub fn merge<'a>(jdts: impl IntoIterator) -> (Self, usize) { + jdts.into_iter() + .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) + } +} + +impl Display for JumpDestTableWitness { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableWitness ===")?; + + for (code, ctxtbls) in &self.0 { + write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; + } + Ok(()) + } +} + +impl Display for ContextJumpDests { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let v: Vec<_> = self.0.iter().sorted().collect(); + for (ctx, offsets) in v.into_iter() { + write!(f, " ctx: {:>4}: [", ctx)?; + for offset in offsets { + write!(f, "{:#}, ", offset)?; + } + writeln!(f, "]")?; + } + Ok(()) + } +} + +impl Display for JumpDestTableProcessed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableProcessed ===")?; + + let v = sorted(self.0.clone()); + for (ctx, code) in v { + writeln!(f, "ctx: {:?} {:?}", ctx, code)?; + } + Ok(()) + } +} + +impl Deref for ContextJumpDests { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ContextJumpDests { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableProcessed { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableProcessed { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableWitness { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableWitness { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(test)] +mod test { + use keccak_hash::H256; + + use super::JumpDestTableWitness; + + #[test] + fn test_extend() { + let code_hash = H256::default(); + + let mut table1 = JumpDestTableWitness::default(); + table1.insert(code_hash, 1, 1); + table1.insert(code_hash, 2, 2); + table1.insert(code_hash, 42, 3); + table1.insert(code_hash, 43, 4); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let mut expected = JumpDestTableWitness::default(); + expected.insert(code_hash, 1, 1); + expected.insert(code_hash, 2, 2); + expected.insert(code_hash, 42, 3); + expected.insert(code_hash, 43, 4); + expected.insert(code_hash, 44, 1); + expected.insert(code_hash, 45, 2); + expected.insert(code_hash, 85, 3); + expected.insert(code_hash, 86, 4); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } +} diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 9c7625d2b..87d9ed1f7 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use anyhow::anyhow; use ethereum_types::{Address, BigEndianHash, H256, U256}; +use jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use log::log_enabled; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; @@ -32,6 +33,7 @@ use crate::util::{h2u, u256_to_usize}; use crate::witness::memory::{MemoryAddress, MemoryChannel, MemoryState}; use crate::witness::state::RegistersState; +pub mod jumpdest; pub(crate) mod linked_list; pub mod mpt; pub(crate) mod prover_input; @@ -106,6 +108,10 @@ pub struct GenerationInputs { /// /// This is specific to `cdk-erigon`. pub ger_data: Option<(H256, H256)>, + + /// A table listing each JUMPDESTs reached in each call context under + /// associated code hash. + pub jumpdest_tables: Vec>, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -156,6 +162,10 @@ pub struct TrimmedGenerationInputs { /// The hash of the current block, and a list of the 256 previous block /// hashes. pub block_hashes: BlockHashes, + + /// A list of tables listing each JUMPDESTs reached in each call context + /// under associated code hash. + pub jumpdest_tables: Vec>, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -230,6 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), + jumpdest_tables: self.jumpdest_tables.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 16c1e5310..ae49366b3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -6,10 +6,12 @@ use std::str::FromStr; use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; +use keccak_hash::keccak; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; +use super::jumpdest::{JumpDestTableProcessed, JumpDestTableWitness}; use super::linked_list::{ LinkedList, ACCOUNTS_LINKED_LIST_NODE_SIZE, STORAGE_LINKED_LIST_NODE_SIZE, }; @@ -20,7 +22,9 @@ use crate::cpu::kernel::constants::cancun_constants::{ POINT_EVALUATION_PRECOMPILE_RETURN_VALUE, }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; -use crate::cpu::kernel::interpreter::simulate_cpu_and_get_user_jumps; +use crate::cpu::kernel::interpreter::{ + set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, +}; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; use crate::generation::prover_input::EvmField::{ @@ -38,6 +42,10 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; +/// A set of contract code as a byte arrays. From this a mapping: hash -> +/// contract can be built. +pub type CodeDb = BTreeSet>; + /// Prover input function represented as a scoped function name. /// Example: `PROVER_INPUT(ff::bn254_base::inverse)` is represented as /// `ProverInputFn([ff, bn254_base, inverse])`. @@ -364,12 +372,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - jumpdest_table.remove(&context); + (*jumpdest_table).remove(&context); Ok(U256::zero()) } } @@ -383,7 +391,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) @@ -757,7 +765,22 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = simulate_cpu_and_get_user_jumps("terminate_common", self); + + dbg!(&self.inputs.jumpdest_tables); + eprintln!("Generating JUMPDEST tables"); + // w for witness + let txn_idx = self.next_txn_index - 1; + let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); + let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + + if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { + if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + println!("SIMW {}", simw.clone()); + println!("RPCW {}", rpcw.unwrap()); + assert_eq!(simw.clone(), *rpcw.unwrap()); + } + } + self.jumpdest_table = rpc; Ok(()) } @@ -768,18 +791,23 @@ impl GenerationState { pub(crate) fn set_jumpdest_analysis_inputs( &mut self, jumpdest_table: HashMap>, - ) { - self.jumpdest_table = Some(HashMap::from_iter(jumpdest_table.into_iter().map( - |(ctx, jumpdest_table)| { + ) -> JumpDestTableWitness { + let mut jdtw = JumpDestTableWitness::default(); + self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( + jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + for offset in jumpdest_table.clone() { + jdtw.insert(keccak(code.clone()), ctx, offset); + } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); (ctx, proofs) } else { (ctx, vec![]) } - }, + }), ))); + jdtw } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { @@ -850,7 +878,7 @@ impl GenerationState { /// for which none of the previous 32 bytes in the code (including opcodes /// and pushed bytes) is a PUSHXX and the address is in its range. It returns /// a vector of even size containing proofs followed by their addresses. -fn get_proofs_and_jumpdests( +pub(crate) fn get_proofs_and_jumpdests( code: &[u8], largest_address: usize, jumpdest_table: std::collections::BTreeSet, diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index b094c15f9..39744ddf2 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -8,6 +8,7 @@ use keccak_hash::keccak; use log::Level; use plonky2::hash::hash_types::RichField; +use super::jumpdest::JumpDestTableProcessed; use super::linked_list::{AccountsLinkedList, StorageLinkedList}; use super::mpt::TrieRootPtrs; use super::segments::GenerationSegmentData; @@ -374,7 +375,7 @@ pub struct GenerationState { /// "proof" for a jump destination is either 0 or an address i > 32 in /// the code (not necessarily pointing to an opcode) such that for every /// j in [i, i+32] it holds that code[j] < 0x7f - j + i. - pub(crate) jumpdest_table: Option>>, + pub(crate) jumpdest_table: Option, /// Each entry contains the pair (key, ptr) where key is the (hashed) key /// of an account in the accounts linked list, and ptr is the respective diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index ca2cf2bd7..f813b18cf 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -210,6 +210,9 @@ pub mod verifier; pub mod generation; pub mod witness; +pub use generation::jumpdest; +pub use generation::prover_input::CodeDb; + // Utility modules pub mod curve_pairings; pub mod extension_tower; diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5b8290229..51ac662df 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,6 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 2c1a56829..f13f541b4 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,6 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 170a19df4..6386dbe69 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,6 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/global_exit_root.rs b/evm_arithmetization/tests/global_exit_root.rs index a41d79e6f..8c18b8e47 100644 --- a/evm_arithmetization/tests/global_exit_root.rs +++ b/evm_arithmetization/tests/global_exit_root.rs @@ -114,6 +114,7 @@ fn test_global_exit_root() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + batch_jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 44453ae22..d9fe6f234 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,6 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 7d8ecba5b..27d68af7b 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,6 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 7dbf91399..4109ae09c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,6 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index fcdd73f49..3dfe13e43 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,6 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh new file mode 100755 index 000000000..49f88ed12 --- /dev/null +++ b/test_jerigon.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +#export CARGO_LOG=cargo::core::compiler::fingerprint=debug +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against jergion, current revision: $GITHASH." + +TESTNETBLOCKS=" +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +" + + +KNOWNFAILED=" +28 +444 +" + + + +# 470..663 from Robin +for i in {470..663} +do + ROBIN+=" $i" +done + +# Pick random blocks +for i in {1..10} +do + RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" +done + +# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" + +TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +SHUF=`shuf -e $TESTNETBLOCKS` +echo $SHUF + + + +for BLOCK in $TESTNETBLOCKS; do + GITHASH=`git rev-parse --short HEAD` + WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Checking block $BLOCK" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt +done + +exit 0 diff --git a/test_native.sh b/test_native.sh new file mode 100755 index 000000000..913f27f13 --- /dev/null +++ b/test_native.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + + + +MAINNETBLOCKS=" +20548415 +20240058 +19665756 +20634472 +19807080 +20634403 +19096840 +19240700 +" + +CANCUN=19426587 +TIP=`cast block-number --rpc-url $RPC` +STATICTIP=20721266 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against mainnet, current revision: $GITHASH." + +#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" +BLOCKS="$MAINNETBLOCKS" +echo "Testing blocks: $BLOCKS" + +echo "Downloading witnesses.." + +for BLOCK in $BLOCKS; do + WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" + until [ -f $WITNESS -a -s $WITNESS ]; do + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + EXITCODE=$? + + if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + then + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt + break + fi + + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt + done + + echo "Witness for block $BLOCK ($WITNESS) prepared." +done + +echo "Finished downloading witnesses." +echo "Testing prepared witnesses.." + +for WITNESS in witnesses/*.native.$GITHASH.witness.json; do + echo "Testing $WITNESS" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +done + +echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 64c0dc2ee..243bacf29 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -10,6 +11,7 @@ use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; use evm_arithmetization::{ generation::{mpt::AccountRlp, TrieInputs}, + jumpdest::JumpDestTableWitness, proof::TrieRoots, testing_utils::{BEACON_ROOTS_CONTRACT_ADDRESS, HISTORY_BUFFER_LENGTH}, GenerationInputs, @@ -87,6 +89,7 @@ pub fn entrypoint( }, after, withdrawals, + jumpdest_tables, }| GenerationInputs:: { txn_number_before: first_txn_ix.into(), gas_used_before: running_gas_used.into(), @@ -113,6 +116,7 @@ pub fn entrypoint( block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, + jumpdest_tables, }, ) .collect()) @@ -256,6 +260,8 @@ struct Batch { /// Empty for all but the final batch pub withdrawals: Vec<(Address, U256)>, + + pub jumpdest_tables: Vec>, } /// [`evm_arithmetization::generation::TrieInputs`], @@ -336,6 +342,8 @@ fn middle( )?; } + let mut jumpdest_tables = vec![]; + for txn in batch { let do_increment_txn_ix = txn.is_some(); let TxnInfo { @@ -345,6 +353,7 @@ fn middle( byte_code, new_receipt_trie_node_byte, gas_used: txn_gas_used, + jumpdest_table, }, } = txn.unwrap_or_default(); @@ -475,6 +484,8 @@ fn middle( // the transaction calling them reverted. } + jumpdest_tables.push(jumpdest_table); + if do_increment_txn_ix { txn_ix += 1; } @@ -527,6 +538,7 @@ fn middle( transactions_root: transaction_trie.root(), receipts_root: receipt_trie.root(), }, + jumpdest_tables, }); } // batch in batches diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 901fef89a..5c72d6b3f 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -5,8 +5,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use ethereum_types::{Address, U256}; -use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; -use keccak_hash::H256; +use evm_arithmetization::{ + jumpdest::JumpDestTableWitness, + proof::{BlockHashes, BlockMetadata}, +}; +use keccak_hash::{keccak, H256}; use mpt_trie::partial_trie::HashedPartialTrie; use plonky2::hash::hash_types::NUM_HASH_OUT_ELTS; use serde::{Deserialize, Serialize}; @@ -113,6 +116,9 @@ pub struct TxnMeta { /// Gas used by this txn (Note: not cumulative gas used). pub gas_used: u64, + + /// JumpDest table + pub jumpdest_table: Option, } /// A "trace" specific to an account for a txn. @@ -164,6 +170,16 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } +// Question: Why has this has been removed upstream. Proably unused. +impl ContractCodeUsage { + /// Get code hash from a read or write operation of contract code. + pub fn get_code_hash(&self) -> H256 { + match self { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } + } +} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/trace_decoder/src/jumpdest.rs @@ -0,0 +1 @@ + diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index fba82d8f7..75146fe23 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -62,6 +62,7 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 +mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero_bin/prover/src/cli.rs b/zero_bin/prover/src/cli.rs index 694194ad9..ba3147f5f 100644 --- a/zero_bin/prover/src/cli.rs +++ b/zero_bin/prover/src/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index c694335d4..0215090eb 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -35,6 +35,9 @@ prover = { workspace = true } trace_decoder = { workspace = true } zero_bin_common = { workspace = true } zk_evm_common = { workspace = true } +alloy-serde = { workspace = true } +alloy-primitives = { workspace = true } +keccak-hash = { workspace = true } [build-dependencies] anyhow = { workspace = true } diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 00d56cf48..3a9f7f866 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,12 +1,29 @@ -use alloy::{providers::Provider, rpc::types::eth::BlockId, transports::Transport}; +use std::collections::BTreeMap; +use std::ops::Deref as _; + +use alloy::providers::ext::DebugApi; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::{ + providers::Provider, + rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + transports::Transport, +}; +use alloy_primitives::Address; use anyhow::Context as _; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use futures::stream::FuturesOrdered; +use futures::StreamExt as _; use prover::BlockProverInput; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; +use trace_decoder::{ + BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, +}; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; +use crate::jumpdest; +use crate::jumpdest::structlogprime::try_reserialize; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -33,16 +50,63 @@ where "debug_traceBlockByNumber".into(), (target_block_id, json!({"tracer": "zeroTracer"})), ) - .await?; + .await? + .into_iter() + .map(|ztr| ztr.result) + .collect::>(); // Grab block witness info (packed as combined trie pre-images) - let block_witness = cached_provider .get_provider() .await? .raw_request::<_, String>("eth_getWitness".into(), vec![target_block_id]) .await?; + let block = cached_provider + .get_block(target_block_id, BlockTransactionsKind::Full) + .await?; + + let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results + .iter() + .map(|TxnInfo { traces, meta: _ }| traces) + .collect::>(); + + let jdts: Vec> = process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + &tx_traces, + ) + .await?; + + // weave in the JDTs + let txn_info = tx_results + .into_iter() + .zip(jdts) + .map( + |( + TxnInfo { + traces, + meta: + TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: _, + }, + }, + jdt, + )| TxnInfo { + traces, + meta: TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: jdt, + }, + }, + ) + .collect(); + let other_data = fetch_other_block_data(cached_provider, target_block_id, checkpoint_block_number).await?; @@ -53,9 +117,69 @@ where compact: hex::decode(block_witness.strip_prefix("0x").unwrap_or(&block_witness)) .context("invalid hex returned from call to eth_getWitness")?, }), - txn_info: tx_results.into_iter().map(|it| it.result).collect(), + txn_info, code_db: Default::default(), }, other_data, }) } + +/// Processes the transactions in the given block and updates the code db. +pub async fn process_transactions( + block: &Block, + provider: &ProviderT, + tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], +) -> anyhow::Result>> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let futures = block + .transactions + .as_transactions() + .context("No transactions in block")? + .iter() + .zip(tx_traces) + .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .collect::>(); + + let vec_of_res = futures.collect::>().await; + vec_of_res.into_iter().collect::, _>>() +} + +/// Processes the transaction with the given transaction hash and updates the +/// accounts state. +pub async fn process_transaction( + provider: &ProviderT, + tx: &Transaction, + tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, +) -> anyhow::Result> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let structlog_trace = provider + .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) + .await?; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let tx_traces = tx_trace + .iter() + .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) + .collect(); + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + Ok(jumpdest_table) +} diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs new file mode 100644 index 000000000..7e9bde277 --- /dev/null +++ b/zero_bin/rpc/src/jumpdest.rs @@ -0,0 +1,417 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; +use std::ops::Not as _; +use std::sync::OnceLock; + +use __compat_primitive_types::H256; +use alloy::primitives::Address; +use alloy::primitives::U160; +use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy_primitives::U256; +use anyhow::ensure; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use keccak_hash::keccak; +use trace_decoder::TxnTrace; +use tracing::trace; + +/// Tracing options for the `debug_traceTransaction` call to get structlog. +/// Used for filling JUMPDEST table. +pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { + GethDebugTracingOptions { + config: GethDefaultTracingOptions { + disable_stack: Some(false), + // needed for CREATE2 + disable_memory: Some(false), + disable_storage: Some(true), + ..GethDefaultTracingOptions::default() + }, + tracer: None, + ..GethDebugTracingOptions::default() + } +} + +/// Provides a way to check in constant time if an address points to a +/// precompile. +fn precompiles() -> &'static HashSet
{ + static PRECOMPILES: OnceLock> = OnceLock::new(); + PRECOMPILES.get_or_init(|| { + HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) + }) +} + +/// Generate at JUMPDEST table by simulating the call stack in EVM, +/// using a Geth structlog as input. +pub(crate) fn generate_jumpdest_table( + tx: &Transaction, + struct_log: &[StructLog], + tx_traces: &BTreeMap, +) -> anyhow::Result { + trace!("Generating JUMPDEST table for tx: {}", tx.hash); + ensure!(struct_log.is_empty().not(), "Structlog is empty."); + + let mut jumpdest_table = JumpDestTableWitness::default(); + + let callee_addr_to_code_hash: HashMap = tx_traces + .iter() + .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) + .filter(|(_callee_addr, code_usage)| code_usage.is_some()) + .map(|(callee_addr, code_usage)| { + (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + }) + .collect(); + + trace!( + "Transaction: {} is a {}.", + tx.hash, + if tx.to.is_some() { + "message call" + } else { + "contract creation" + } + ); + + let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { + // Guard against transactions to a non-contract address. + ensure!( + callee_addr_to_code_hash.contains_key(&to_address), + format!("Callee addr {} is not at contract address", to_address) + ); + callee_addr_to_code_hash[&to_address] + } else { + let init = tx.input.clone(); + keccak(init) + }; + + // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with + // true condition and target `jump_target`. + let mut prev_jump = None; + + // Call depth of the previous `entry`. We initialize to 0 as this compares + // smaller to 1. + //let mut prev_depth = 0; + // The next available context. Starts at 1. Never decrements. + let mut next_ctx_available = 1; + // Immediately use context 1; + let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; + next_ctx_available += 1; + + for (step, entry) in struct_log.iter().enumerate() { + let op = entry.op.as_str(); + let curr_depth: usize = entry.depth.try_into().unwrap(); + + ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); + + // ensure!(call_stack.is_empty().not(), "Call stack was empty."); + while curr_depth < call_stack.len() { + call_stack.pop(); + } + + let (code_hash, ctx) = call_stack.last().unwrap(); + + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); + + match op { + "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; // actually 6 or 7. + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 + let [_gas, address, ..] = evm_stack[..] else { + unreachable!() + }; + + let callee_address = { + // Clear the upper half of the operand. + let callee_raw = *address; + // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); + // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); + + ensure!(callee_raw <= U256::from(U160::MAX)); + let lower_20_bytes = U160::from(callee_raw); + Address::from(lower_20_bytes) + }; + + if precompiles().contains(&callee_address) { + trace!("Called precompile at address {}.", &callee_address); + } else if callee_addr_to_code_hash.contains_key(&callee_address) { + let code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((code_hash, next_ctx_available)); + } else { + // This case happens if calling an EOA. This is described + // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun + trace!( + "Callee address {} has no associated `code_hash`.", + &callee_address + ); + } + next_ctx_available += 1; + prev_jump = None; + } + "CREATE" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 3; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + unreachable!() + }; + + let contract_address = tx.from.create(tx.nonce); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "CREATE2" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 4; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, offset, size, salt, ..] = evm_stack[..] else { + unreachable!() + }; + ensure!(*offset < U256::from(usize::MAX)); + let offset: usize = offset.to(); + ensure!(*size < U256::from(usize::MAX)); + let size: usize = size.to(); + let salt: [u8; 32] = salt.to_be_bytes(); + + ensure!( + size == 0 + || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), + "No or insufficient memory available for {op}." + ); + let memory_raw: &[String] = entry.memory.as_ref().unwrap(); + let memory: Vec = memory_raw + .iter() + .flat_map(|s| { + let c = s.parse(); + // ensure!(c.is_ok(), "No memory."); + let a: U256 = c.unwrap(); + let d: [u8; 32] = a.to_be_bytes(); + d + }) + .collect(); + let init_code = &memory[offset..offset + size]; + let contract_address = tx.from.create2_from_code(salt, init_code); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "JUMP" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 1; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [counter, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = counter.to::(); + + prev_jump = Some(jump_target); + } + "JUMPI" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [pc, condition, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = pc.to::(); + let jump_condition = condition.is_zero().not(); + + prev_jump = if jump_condition { + Some(jump_target) + } else { + None + }; + } + "JUMPDEST" => { + let jumped_here = if let Some(jmp_target) = prev_jump { + jmp_target == entry.pc + } else { + false + }; + let jumpdest_offset = entry.pc as usize; + if jumped_here { + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + } + // else: we do not care about JUMPDESTs reached through fall-through. + prev_jump = None; + } + "EXTCODECOPY" | "EXTCODESIZE" => { + next_ctx_available += 1; + prev_jump = None; + } + // "RETURN" | "REVERT" | "STOP" => { + // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); + // do_pop = true; + // prev_jump = None; + // } + // "SELFDESTRUCT" => { + // do_pop = true; + // prev_jump = None; + // } + _ => { + prev_jump = None; + } + } + } + Ok(jumpdest_table) +} + +pub mod structlogprime { + use core::option::Option::None; + use std::collections::BTreeMap; + + use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy_primitives::{Bytes, B256, U256}; + use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; + use serde_json::Value; + + /// Geth Default struct log trace frame + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub(crate) struct DefaultFramePrime { + /// Whether the transaction failed + pub failed: bool, + /// How much gas was used. + pub gas: u64, + /// Output of the transaction + #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] + pub return_value: Bytes, + /// Recorded traces of the transaction + pub struct_logs: Vec, + } + + /// Represents a struct log entry in a trace + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + pub(crate) struct StructLogPrime { + /// Program counter + pub pc: u64, + /// Opcode to be executed + pub op: String, + /// Remaining gas + pub gas: u64, + /// Cost for executing op + #[serde(rename = "gasCost")] + pub gas_cost: u64, + /// Current call depth + pub depth: u64, + /// Error message if any + #[serde(default, skip)] + pub error: Option, + /// EVM stack + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stack: Option>, + /// Last call's return data. Enabled via enableReturnData + #[serde( + default, + rename = "returnData", + skip_serializing_if = "Option::is_none" + )] + pub return_data: Option, + /// ref + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option>, + /// Size of memory. + #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] + pub memory_size: Option, + /// Storage slots of current contract read from and written to. Only + /// emitted for SLOAD and SSTORE. Disabled via disableStorage + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_string_storage_map_opt" + )] + pub storage: Option>, + /// Refund counter + #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] + pub refund_counter: Option, + } + + /// Serializes a storage map as a list of key-value pairs _without_ + /// 0x-prefix + pub(crate) fn serialize_string_storage_map_opt( + storage: &Option>, + s: S, + ) -> Result { + match storage { + None => s.serialize_none(), + Some(storage) => { + let mut m = s.serialize_map(Some(storage.len()))?; + for (key, val) in storage.iter() { + let key = format!("{:?}", key); + let val = format!("{:?}", val); + // skip the 0x prefix + m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; + } + m.end() + } + } + } + + impl TryInto for DefaultFramePrime { + fn try_into(self) -> Result { + let a = serde_json::to_string(&self)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let c = serde_json::to_string(&b)?; + let d: DefaultFrame = serde_json::from_str(&c)?; + Ok(d) + } + + type Error = anyhow::Error; + } + + pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { + let a = serde_json::to_string(&structlog_object)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let d: DefaultFrame = b.try_into()?; + Ok(d) + } +} diff --git a/zero_bin/rpc/src/lib.rs b/zero_bin/rpc/src/lib.rs index 87a581a14..d54d1b88d 100644 --- a/zero_bin/rpc/src/lib.rs +++ b/zero_bin/rpc/src/lib.rs @@ -21,6 +21,7 @@ use trace_decoder::{BlockLevelData, OtherBlockData}; use tracing::warn; pub mod jerigon; +pub mod jumpdest; pub mod native; pub mod retry; diff --git a/zero_bin/rpc/src/main.rs b/zero_bin/rpc/src/main.rs index 12594877a..e9ddd0122 100644 --- a/zero_bin/rpc/src/main.rs +++ b/zero_bin/rpc/src/main.rs @@ -217,6 +217,9 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::Registry::default() .with( tracing_subscriber::fmt::layer() + // With the default configuration trace information is written + // to stdout, but we already use to write our payload (the witness). + .with_writer(std::io::stderr) .with_ansi(false) .compact() .with_filter(EnvFilter::from_default_env()), diff --git a/zero_bin/rpc/src/native/mod.rs b/zero_bin/rpc/src/native/mod.rs index 2e9527274..f128e6d6c 100644 --- a/zero_bin/rpc/src/native/mod.rs +++ b/zero_bin/rpc/src/native/mod.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::ops::Deref; use std::sync::Arc; @@ -15,7 +14,8 @@ use zero_bin_common::provider::CachedProvider; mod state; mod txn; -type CodeDb = BTreeSet>; +pub use txn::process_transaction; +pub use txn::process_transactions; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( @@ -39,7 +39,7 @@ where } /// Processes the block with the given block number and returns the block trace. -async fn process_block_trace( +pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, ) -> anyhow::Result diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 5e3be656a..10a110e83 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -9,25 +9,27 @@ use alloy::{ Provider, }, rpc::types::{ - eth::Transaction, - eth::{AccessList, Block}, + eth::{AccessList, Block, Transaction}, trace::geth::{ - AccountState, DiffMode, GethDebugBuiltInTracerType, GethTrace, PreStateConfig, - PreStateFrame, PreStateMode, + AccountState, DiffMode, GethDebugBuiltInTracerType, GethDebugTracerType, + GethDebugTracingOptions, GethTrace, PreStateConfig, PreStateFrame, PreStateMode, + StructLog, }, - trace::geth::{GethDebugTracerType, GethDebugTracingOptions}, }, transports::Transport, }; use anyhow::Context as _; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use super::CodeDb; -use crate::Compat; +use crate::{ + jumpdest::{self, structlogprime::try_reserialize}, + Compat, +}; /// Processes the transactions in the given block and updates the code db. -pub(super) async fn process_transactions( +pub async fn process_transactions( block: &Block, provider: &ProviderT, ) -> anyhow::Result<(CodeDb, Vec)> @@ -55,7 +57,7 @@ where /// Processes the transaction with the given transaction hash and updates the /// accounts state. -async fn process_transaction( +pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, ) -> anyhow::Result<(CodeDb, TxnInfo)> @@ -63,17 +65,12 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace) = fetch_tx_data(provider, &tx.hash).await?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let tx_meta = TxnMeta { - byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), - new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), - gas_used: tx_receipt.gas_used as u64, - }; - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), @@ -85,7 +82,30 @@ where // Handle case when transaction failed and a contract creation was reverted if !tx_status && tx_receipt.contract_address.is_some() { tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); - } + }; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + // if jumpdest_table.is_some() { eprintln!("======================> 1")}; + + let tx_meta = TxnMeta { + byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), + new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), + gas_used: tx_receipt.gas_used as u64, + jumpdest_table, + }; Ok(( code_db, @@ -103,7 +123,12 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, -) -> anyhow::Result<(::ReceiptResponse, GethTrace, GethTrace), anyhow::Error> +) -> anyhow::Result<( + ::ReceiptResponse, + GethTrace, + GethTrace, + GethTrace, +)> where ProviderT: Provider, TransportT: Transport + Clone, @@ -111,14 +136,21 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); + let structlog_trace_fut = + provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut + )?; Ok(( tx_receipt.context("Transaction receipt not found.")?, pre_trace, diff_trace, + structlog_trace, )) } diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 815a7048d..f3d128d46 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -17,24 +17,25 @@ else num_procs=$(nproc) fi -# Force the working directory to always be the `tools/` directory. +# Force the working directory to always be the `tools/` directory. TOOLS_DIR=$(dirname $(realpath "$0")) PROOF_OUTPUT_DIR="${TOOLS_DIR}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" -echo "Block batch size: $BLOCK_BATCH_SIZE" +BATCH_SIZE=${BATCH_SIZE:=1} +echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${TOOLS_DIR}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${TOOLS_DIR}/test.out" + # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -export TOKIO_WORKER_THREADS=$num_procs +#export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 -export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_MIN_STACK=33554432 +#export RUST_BACKTRACE=full +#export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -95,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $TEST_OUT_PATH + cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH @@ -108,10 +109,9 @@ fi cargo build --release --jobs "$num_procs" - start_time=$(date +%s%N) -"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ - --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG +"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE \ + --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG end_time=$(date +%s%N) set +o pipefail From 459148215f9e3b92d8a041ae92a1ec4eceecf296 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 20:26:10 +0200 Subject: [PATCH 02/53] feedback + cleanups --- zero_bin/rpc/src/jumpdest.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7e9bde277..2a35cc877 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -105,11 +105,11 @@ pub(crate) fn generate_jumpdest_table( ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); - // ensure!(call_stack.is_empty().not(), "Call stack was empty."); while curr_depth < call_stack.len() { call_stack.pop(); } + ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -179,6 +179,7 @@ pub(crate) fn generate_jumpdest_table( }; let contract_address = tx.from.create(tx.nonce); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -202,26 +203,30 @@ pub(crate) fn generate_jumpdest_table( let offset: usize = offset.to(); ensure!(*size < U256::from(usize::MAX)); let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); ensure!( - size == 0 - || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), - "No or insufficient memory available for {op}." + entry.memory.is_some() && size <= memory_size, + "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory: Vec = memory_raw + let memory_parsed: Vec> = memory_raw .iter() - .flat_map(|s| { + .map(|s| { let c = s.parse(); - // ensure!(c.is_ok(), "No memory."); + ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); let d: [u8; 32] = a.to_be_bytes(); - d + Ok(d) }) .collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let memory: Vec = mem_res?.concat(); + let init_code = &memory[offset..offset + size]; let contract_address = tx.from.create2_from_code(salt, init_code); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -284,15 +289,6 @@ pub(crate) fn generate_jumpdest_table( next_ctx_available += 1; prev_jump = None; } - // "RETURN" | "REVERT" | "STOP" => { - // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); - // do_pop = true; - // prev_jump = None; - // } - // "SELFDESTRUCT" => { - // do_pop = true; - // prev_jump = None; - // } _ => { prev_jump = None; } From 91c29454e9509ad13eb5e5235e84eae967575132 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 21:35:55 +0200 Subject: [PATCH 03/53] cleanups --- test_jerigon.sh | 9 +++++---- zero_bin/rpc/src/jerigon.rs | 2 ++ zero_bin/rpc/src/native/txn.rs | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index 49f88ed12..fe335869b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -60,17 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF +echo "Testing: $BLOCKS" -for BLOCK in $TESTNETBLOCKS; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` - WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3a9f7f866..2fefd90ec 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -19,6 +19,7 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; +use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; @@ -177,6 +178,7 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 10a110e83..f0be69ebf 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -22,6 +22,7 @@ use anyhow::Context as _; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; +use tracing::debug; use crate::{ jumpdest::{self, structlogprime::try_reserialize}, @@ -94,12 +95,11 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); - // if jumpdest_table.is_some() { eprintln!("======================> 1")}; - let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From b58c5d67cfd4f0e7948e6a2419318ce5e0a40f25 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 23:08:47 +0200 Subject: [PATCH 04/53] fix overflow --- zero_bin/rpc/src/jumpdest.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 2a35cc877..b4f048f28 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -199,9 +199,9 @@ pub(crate) fn generate_jumpdest_table( let [_value, offset, size, salt, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset < U256::from(usize::MAX)); + ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); - ensure!(*size < U256::from(usize::MAX)); + ensure!(*size <= U256::from(usize::MAX)); let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); @@ -246,7 +246,8 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = counter.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); } @@ -260,10 +261,11 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [pc, condition, ..] = evm_stack[..] else { + let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = pc.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { From 037fb5710c5fd66888f1fe1f4a00960c5eddca03 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 10:58:36 +0200 Subject: [PATCH 05/53] fmt --- zero_bin/rpc/src/jumpdest.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index b4f048f28..7ea971e13 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -246,7 +246,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); @@ -264,7 +267,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); From 85ee8c274b44a7506c7e21732c88993c76e88fe3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:44:37 +0200 Subject: [PATCH 06/53] fix testscripts --- test_jerigon.sh | 18 ++++++++++-------- test_native.sh | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index fe335869b..deac74b2b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -15,7 +15,7 @@ export TOKIO_WORKER_THREADS=4 export RUST_BACKTRACE=full export RUST_LOG=info export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -60,16 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` -BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF -echo "Testing: $BLOCKS" +#echo "Testing: $BLOCKS" +printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt +printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in $BLOCKS; do +for BLOCK in {1..256}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" @@ -77,13 +79,13 @@ for BLOCK in $BLOCKS; do echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else RESULT="failure" fi - printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt + printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done exit 0 diff --git a/test_native.sh b/test_native.sh index 913f27f13..fcb85fba5 100755 --- a/test_native.sh +++ b/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail export RPC= if [ -z $RPC ]; then @@ -49,10 +49,10 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" until [ -f $WITNESS -a -s $WITNESS ]; do echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS EXITCODE=$? - if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break @@ -71,7 +71,7 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else From 1243768b0b026305d326394903bed23c5ba8b546 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:45:24 +0200 Subject: [PATCH 07/53] refactor --- zero_bin/rpc/src/jerigon.rs | 34 ++++++----------- zero_bin/rpc/src/jumpdest.rs | 68 +++++++++++++++++++++++++++++----- zero_bin/rpc/src/native/txn.rs | 32 +++++++--------- 3 files changed, 84 insertions(+), 50 deletions(-) diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 2fefd90ec..3c36f7454 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,11 +1,9 @@ use std::collections::BTreeMap; use std::ops::Deref as _; -use alloy::providers::ext::DebugApi; -use alloy::rpc::types::trace::geth::StructLog; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -23,8 +21,7 @@ use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; -use crate::jumpdest; -use crate::jumpdest::structlogprime::try_reserialize; +use crate::jumpdest::{self, get_normalized_structlog}; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -159,28 +156,21 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let structlog_trace = provider - .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) - .await?; - - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - let tx_traces = tx_trace .iter() .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { - jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + + let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); Ok(jumpdest_table) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7ea971e13..97531eee9 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -7,25 +7,31 @@ use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; use alloy::primitives::U160; +use alloy::providers::ext::DebugApi; +use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy::transports::RpcError; +use alloy::transports::Transport; +use alloy::transports::TransportErrorKind; +use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use structlogprime::normalize_structlog; use trace_decoder::TxnTrace; use tracing::trace; -/// Tracing options for the `debug_traceTransaction` call to get structlog. -/// Used for filling JUMPDEST table. -pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { +/// Pass `true` for the components needed. +fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { GethDebugTracingOptions { config: GethDefaultTracingOptions { - disable_stack: Some(false), + disable_stack: Some(!stack), // needed for CREATE2 - disable_memory: Some(false), - disable_storage: Some(true), + disable_memory: Some(!memory), + disable_storage: Some(!storage), ..GethDefaultTracingOptions::default() }, tracer: None, @@ -33,6 +39,38 @@ pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { } } +fn trace_contains_create2(structlog: Vec) -> bool { + structlog.iter().any(|entry| entry.op == "CREATE2") +} + +// Gets the lightest possible structlog for transcation `tx_hash`. +pub(crate) async fn get_normalized_structlog( + provider: &ProviderT, + tx_hash: &B256, +) -> Result>, RpcError> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let light_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + .await?; + + let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + + let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + + let structlog = provider + .debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ) + .await?; + + let ret = normalize_structlog(structlog).await; + Ok(ret) +} + /// Provides a way to check in constant time if an address points to a /// precompile. fn precompiles() -> &'static HashSet
{ @@ -309,7 +347,7 @@ pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; - use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; use alloy_primitives::{Bytes, B256, U256}; use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; use serde_json::Value; @@ -412,10 +450,22 @@ pub mod structlogprime { type Error = anyhow::Error; } - pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { - let a = serde_json::to_string(&structlog_object)?; + pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { + let a = serde_json::to_string(structlog_object)?; let b: DefaultFramePrime = serde_json::from_str(&a)?; let d: DefaultFrame = b.try_into()?; Ok(d) } + + pub(crate) async fn normalize_structlog( + unnormalized_structlog: GethTrace, + ) -> Option> { + match unnormalized_structlog { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + } + } } diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index f0be69ebf..80d2f367b 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -18,14 +18,14 @@ use alloy::{ }, transports::Transport, }; -use anyhow::Context as _; +use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::{ - jumpdest::{self, structlogprime::try_reserialize}, + jumpdest::{self, get_normalized_structlog}, Compat, }; @@ -66,7 +66,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + let (tx_receipt, pre_trace, diff_trace, structlog_opt) = fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); @@ -85,19 +85,14 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { - jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); let tx_meta = TxnMeta { @@ -127,7 +122,7 @@ async fn fetch_tx_data( ::ReceiptResponse, GethTrace, GethTrace, - GethTrace, + Option>, )> where ProviderT: Provider, @@ -136,8 +131,7 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = - provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( tx_receipt_fut, From e7244c6bf648343800572a10024d91050cfd5cd7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 22:29:01 +0200 Subject: [PATCH 08/53] for testing --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 10 ++++++++-- test_jerigon.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index d0cb867fa..19b1db7c8 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; use keccak_hash::H256; -use log::Level; +use log::{trace, Level}; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -167,7 +167,13 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( let ctx_proofs = (*jumpdest_table_rpc) .iter() .flat_map(|(code_addr, ctx_jumpdests)| { - prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + let code = if code_map.contains_key(code_addr) { + &code_map[code_addr] + } else { + &vec![] + }; + trace!("code: {:?} <============", &code); + prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); JumpDestTableProcessed::new(ctx_proofs) diff --git a/test_jerigon.sh b/test_jerigon.sh index deac74b2b..0bd16b311 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -41,6 +41,13 @@ TESTNETBLOCKS=" KNOWNFAILED=" +2 +15 +28 +35 +37 +43 +65 28 444 " @@ -71,7 +78,7 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {1..256}; do +for BLOCK in {66..688}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" From 16e9c261ed14dcb9ec666863635f61deb6e1716d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:24:29 +0200 Subject: [PATCH 09/53] extract initcode --- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 7 +++- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/src/generation/mod.rs | 4 +- .../src/generation/prover_input.rs | 21 +++++++--- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- test_jerigon.sh | 27 +++++++++++-- trace_decoder/src/core.rs | 39 +++++++++++++++---- 15 files changed, 90 insertions(+), 30 deletions(-) diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 57b902c63..2b35b2a1c 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 19b1db7c8..7bc5e1dcc 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -172,7 +172,12 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( } else { &vec![] }; - trace!("code: {:?} <============", &code); + trace!( + "code: {:?}, code_addr: {:?} <============", + &code, + &code_addr + ); + trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index 1cc5588e1..fbae6647d 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 08fe57b1a..673b516b0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 87d9ed1f7..e5ec7bde3 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index ae49366b3..dc59bda2c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; +use log::trace; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -769,15 +770,18 @@ impl GenerationState { dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); // w for witness - let txn_idx = self.next_txn_index - 1; - let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); - let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + // let txn_idx = self.next_txn_index - 1; + // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + let rpcw = &self.inputs.jumpdest_tables; + let rpc = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.unwrap()); - assert_eq!(simw.clone(), *rpcw.unwrap()); + println!("RPCW {}", rpcw.clone().unwrap()); + assert_eq!(simw.clone(), rpcw.clone().unwrap()); } } self.jumpdest_table = rpc; @@ -796,6 +800,11 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + trace!( + "ctx: {ctx}, code_hash: {:?} code: {:?}", + keccak(code.clone()), + code + ); for offset in jumpdest_table.clone() { jdtw.insert(keccak(code.clone()), ctx, offset); } diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 51ac662df..12a84fbe7 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index f13f541b4..475d03a8e 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 6386dbe69..8d30f85d1 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index d9fe6f234..951ebbb28 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 27d68af7b..221113b26 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 4109ae09c..de8e096cf 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 3dfe13e43..2aa776e54 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh index 0bd16b311..b1021b03b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -48,10 +48,31 @@ KNOWNFAILED=" 37 43 65 + 28 444 -" +43 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +72 +77 +" # 470..663 from Robin @@ -78,11 +99,11 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {66..688}; do +for BLOCK in $KNOWNFAILED; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 243bacf29..2cfb934ed 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,4 +1,4 @@ -use core::option::Option::None; +use core::{convert::Into as _, option::Option::None}; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -6,6 +6,11 @@ use std::{ }; use alloy::primitives::address; +use alloy::{ + consensus::{Transaction, TxEnvelope}, + primitives::TxKind, + rlp::Decodable as _, +}; use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; @@ -97,7 +102,7 @@ pub fn entrypoint( running_gas_used += gas_used; running_gas_used.into() }, - signed_txns: byte_code.into_iter().map(Into::into).collect(), + signed_txns: byte_code.clone().into_iter().map(Into::into).collect(), withdrawals, ger_data: None, tries: TrieInputs { @@ -109,14 +114,34 @@ pub fn entrypoint( trie_roots_after: after, checkpoint_state_trie_root, checkpoint_consolidated_hash, - contract_code: contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(), + contract_code: { + let cc = contract_code.into_iter(); + + let initcodes = + byte_code + .iter() + .filter_map(|nonempty_txn_raw| -> Option> { + let tx_envelope = + TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + match tx_envelope.to() { + TxKind::Create => Some(tx_envelope.input().to_vec()), + TxKind::Call(_address) => None, + } + }); + + cc.chain(initcodes) + .map(|it| (keccak_hash::keccak(&it), it)) + .collect() + }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables, + jumpdest_tables: { + jumpdest_tables + .into_iter() + .collect::>>() + .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + }, }, ) .collect()) From f3871d96717b601852102384f013cae6f683964e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:47:49 +0200 Subject: [PATCH 10/53] improve test script --- test_jerigon.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index b1021b03b..53309228c 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -20,7 +20,7 @@ export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." -TESTNETBLOCKS=" +CIBLOCKS=" 1 2 3 @@ -81,30 +81,28 @@ do ROBIN+=" $i" done -# Pick random blocks -for i in {1..10} -do - RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" -done - -# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` - -SHUF=`shuf -e $TESTNETBLOCKS` -echo $SHUF +TIP=688 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` +# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` +BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt +echo "---------------------------" | tee -a witnesses/jerigon_results.txt -for BLOCK in $KNOWNFAILED; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Checking block $BLOCK" + echo "Testing blocks:" + echo $BLOCKS + echo "Now testing block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] From 4fd6b8b7028c6a9d769b20f612533499018609ff Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 10:35:20 +0200 Subject: [PATCH 11/53] fix stack issue --- zero_bin/rpc/src/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 97531eee9..67c5e68a2 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -212,7 +212,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + let [_value, _offset, _size, ..] = evm_stack[..] else { unreachable!() }; From 88eb73d7db4f49e5f440e3c7d8e875c5343d7cd8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 13:23:07 +0200 Subject: [PATCH 12/53] random fixes --- Cargo.toml | 2 +- .../src/generation/prover_input.rs | 24 +++--- test_jerigon.sh | 77 +++++++++++++++++-- zero_bin/rpc/src/jumpdest.rs | 4 +- 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 84a438806..294681d05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ tiny-keccak = "2.0.2" tokio = { version = "1.38.0", features = ["full"] } toml = "0.8.14" tower = "0.4" -tracing = "0.1" +tracing = { version = "0.1", features = ["attributes"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } u4 = "0.1.0" uint = "0.9.5" diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index dc59bda2c..8751e2ded 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,24 +767,30 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); + dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code let rpcw = &self.inputs.jumpdest_tables; - let rpc = rpcw + let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { - println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.clone().unwrap()); - assert_eq!(simw.clone(), rpcw.clone().unwrap()); - } + let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + + let (sim, simw): (Option, Option) = + sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + trace!("SIMW {}", sw); + trace!("RPCW {}", rw); + assert_eq!(rw, &sw); } - self.jumpdest_table = rpc; + + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index 53309228c..2b1f5c2d5 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -40,7 +40,71 @@ CIBLOCKS=" " -KNOWNFAILED=" +STILLFAIL=" +37 +75 +15 +35 +43 +72 +77 +184 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +477 +478 +444 +" + +JUMPI=" +662 +664 +665 +667 +670 +" + +CONTAINSKEY=" +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +72 +" + +CREATE2=" +43 +566 +77 +" + +DECODING=" +477 +478 +" + +USEDTOFAIL=" 2 15 28 @@ -50,7 +114,6 @@ KNOWNFAILED=" 65 28 -444 43 460 @@ -88,7 +151,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` # CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS="$DECODING" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -99,10 +163,9 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Testing blocks:" - echo $BLOCKS - echo "Now testing block $BLOCK" + timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 67c5e68a2..e31e90bf5 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -82,13 +82,13 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. +#[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, ) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); - ensure!(struct_log.is_empty().not(), "Structlog is empty."); let mut jumpdest_table = JumpDestTableWitness::default(); @@ -123,7 +123,7 @@ pub(crate) fn generate_jumpdest_table( keccak(init) }; - // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. let mut prev_jump = None; From 39cd26c77d5f395fd3d1cf1e4c06e62b69ee2916 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:37:42 +0200 Subject: [PATCH 13/53] fix CREATE2 --- .../src/generation/prover_input.rs | 18 +++-- test_jerigon.sh | 2 + zero_bin/rpc/src/jerigon.rs | 13 ++-- zero_bin/rpc/src/jumpdest.rs | 66 ++++++++++++------- zero_bin/rpc/src/native/txn.rs | 2 +- zero_bin/tools/prove_stdio.sh | 4 +- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 8751e2ded..1919f98d2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,7 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; -use log::trace; +use log::{info, trace}; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -767,7 +767,7 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - eprintln!("Generating JUMPDEST tables"); + info!("Generating JUMPDEST tables"); dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; @@ -775,20 +775,18 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { - trace!("SIMW {}", sw); - trace!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 2b1f5c2d5..5362d6156 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -163,9 +163,11 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." + export RUST_LOG=info zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3c36f7454..0c3900ab8 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -3,7 +3,7 @@ use std::ops::Deref as _; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -161,15 +161,20 @@ where .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + let structlog_opt: Option> = get_normalized_structlog(provider, &tx.hash) + .await + .ok() + .flatten(); let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); None }, - Some, + |jdt|{ + debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); + Some(jdt)}, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index e31e90bf5..321acd9fa 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,3 +1,5 @@ +use core::default::Default; +use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; @@ -21,8 +23,17 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use structlogprime::normalize_structlog; +use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::trace; +use tracing::{instrument, trace}; + +/// The maximum time we are willing to wait for a structlog before failing over +/// to simulating the JumpDest analysis. +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); + +/// Structure of Etheruem memory +type Word = [u8; 32]; +const WORDSIZE: usize = std::mem::size_of::(); /// Pass `true` for the components needed. fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { @@ -52,6 +63,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { + // Optimization: It may be a better default to pull the stack immediately. let light_structlog_trace = provider .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) .await?; @@ -59,16 +71,19 @@ where let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider - .debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ) - .await?; + let structlog = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); - let ret = normalize_structlog(structlog).await; - Ok(ret) + match timeout(TIMEOUT_LIMIT, structlog).await { + Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( + ellapsed_error, + )))), + Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + } } /// Provides a way to check in constant time if an address points to a @@ -119,7 +134,7 @@ pub(crate) fn generate_jumpdest_table( ); callee_addr_to_code_hash[&to_address] } else { - let init = tx.input.clone(); + let init = &tx.input; keccak(init) }; @@ -240,33 +255,36 @@ pub(crate) fn generate_jumpdest_table( ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); ensure!(*size <= U256::from(usize::MAX)); + let size: usize = size.to(); - let memory_size = entry.memory.as_ref().unwrap().len(); - let salt: [u8; 32] = salt.to_be_bytes(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; + let salt: Word = salt.to_be_bytes(); ensure!( - entry.memory.is_some() && size <= memory_size, - "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." + entry.memory.is_some() && offset + size <= memory_size, + "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory_parsed: Vec> = memory_raw + let memory_parsed: Vec> = memory_raw .iter() .map(|s| { - let c = s.parse(); + // let c = s.parse(); + let c = U256::from_str_radix(s, 16); ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); - let d: [u8; 32] = a.to_be_bytes(); + let d: Word = a.to_be_bytes(); Ok(d) }) .collect(); - let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); let memory: Vec = mem_res?.concat(); let init_code = &memory[offset..offset + size]; - let contract_address = tx.from.create2_from_code(salt, init_code); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + let init_code_hash = keccak(init_code); + // let contract_address = tx.from.create2_from_code(salt, init_code); + // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); + // let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; prev_jump = None; @@ -286,7 +304,9 @@ pub(crate) fn generate_jumpdest_table( }; ensure!( *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." + "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", + *counter, + u64::MAX ); let jump_target: u64 = counter.to(); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 80d2f367b..63e61d7e2 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -137,7 +137,7 @@ where tx_receipt_fut, pre_trace_fut, diff_trace_fut, - structlog_trace_fut + structlog_trace_fut, )?; Ok(( diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index f3d128d46..e00e79e8d 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -35,7 +35,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_MIN_STACK=33554432 #export RUST_BACKTRACE=full -#export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -102,7 +102,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." exit 1 fi fi From 8a964b8bd2442c27b1f59c3dd61e3e502f5a80d2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:42:04 +0200 Subject: [PATCH 14/53] fmt, clippy --- .../src/generation/prover_input.rs | 14 ++++++++------ test_jerigon.sh | 2 +- zero_bin/rpc/src/jerigon.rs | 15 +++++++++++---- zero_bin/rpc/src/jumpdest.rs | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1919f98d2..77aaec99d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -775,18 +775,20 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 5362d6156..fd7800872 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -152,7 +152,7 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING" +BLOCKS="$DECODING $CREATE2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 0c3900ab8..82feee7a0 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -169,12 +169,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); + debug!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - |jdt|{ - debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); - Some(jdt)}, + |jdt| { + debug!( + "{:#?}: JumpDestTable generation succeceeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 321acd9fa..8e3ffe8f4 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -249,7 +249,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, offset, size, salt, ..] = evm_stack[..] else { + let [_value, offset, size, _salt, ..] = evm_stack[..] else { unreachable!() }; ensure!(*offset <= U256::from(usize::MAX)); @@ -258,7 +258,7 @@ pub(crate) fn generate_jumpdest_table( let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - let salt: Word = salt.to_be_bytes(); + // let salt: Word = salt.to_be_bytes(); ensure!( entry.memory.is_some() && offset + size <= memory_size, From 32e68bfa33c8ae3c0361ba85c9e61039225a32ae Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 13:46:54 +0200 Subject: [PATCH 15/53] investigate 15,35 --- .../src/generation/prover_input.rs | 5 ++- test_jerigon.sh | 26 ++++++++++-- trace_decoder/src/core.rs | 1 + zero_bin/rpc/src/jerigon.rs | 2 +- zero_bin/rpc/src/jumpdest.rs | 40 ++++++++++--------- zero_bin/tools/prove_stdio.sh | 2 +- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 77aaec99d..c2b1d2169 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,7 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + info!("SIMW {:#?}", sim); + info!("RPCW {:#?}", rpc); + + self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index fd7800872..a1a1c85ca 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -11,9 +11,9 @@ fi mkdir -p witnesses export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info +export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 @@ -137,6 +137,23 @@ USEDTOFAIL=" 77 " +ROUND2=" +15 +35 +566 +664 +665 +667 +670 +" +#444 + +TESTED=" +4 +5 +28 +65 +" # 470..663 from Robin for i in {470..663} @@ -152,7 +169,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING $CREATE2" +#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" +BLOCKS="$ROUND2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -164,7 +182,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2cfb934ed..d227edfb8 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -76,6 +76,7 @@ pub fn entrypoint( withdrawals, )?; + dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 82feee7a0..b3742f27f 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -177,7 +177,7 @@ where }, |jdt| { debug!( - "{:#?}: JumpDestTable generation succeceeded with result: {}", + "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); Some(jdt) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 8e3ffe8f4..4861aeeca 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,4 +1,5 @@ use core::default::Default; +use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; @@ -97,7 +98,7 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -#[instrument] +// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], @@ -107,6 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); + // This does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -126,16 +128,16 @@ pub(crate) fn generate_jumpdest_table( } ); - let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { - // Guard against transactions to a non-contract address. - ensure!( - callee_addr_to_code_hash.contains_key(&to_address), - format!("Callee addr {} is not at contract address", to_address) - ); - callee_addr_to_code_hash[&to_address] - } else { - let init = &tx.input; - keccak(init) + let entrypoint_code_hash: H256 = match tx.to { + Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { + return Ok(jumpdest_table) + } + Some(to_address) => callee_addr_to_code_hash[&to_address], + None => { + let init = &tx.input; + keccak(init) + } }; // `None` encodes that previous `entry` was not a JUMP or JUMPI with true @@ -165,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - trace!("TX: {:?}", tx.hash); - trace!("STEP: {:?}", step); - trace!("STEPS: {:?}", struct_log.len()); - trace!("OPCODE: {}", entry.op.as_str()); - trace!("CODE: {:?}", code_hash); - trace!("CTX: {:?}", ctx); - trace!("CURR_DEPTH: {:?}", curr_depth); - trace!("{:#?}\n", entry); + // trace!("TX: {:?}", tx.hash); + // trace!("STEP: {:?}", step); + // trace!("STEPS: {:?}", struct_log.len()); + // trace!("OPCODE: {}", entry.op.as_str()); + // trace!("CODE: {:?}", code_hash); + // trace!("CTX: {:?}", ctx); + // trace!("CURR_DEPTH: {:?}", curr_depth); + // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index e00e79e8d..88bc359fc 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -96,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH From 184878dda760dd48cbf400d23407e69cacd4fbe0 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 21:08:52 +0200 Subject: [PATCH 16/53] fix scripts --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 6 +++--- scripts/test_native.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index cffb39a34..80515c9e1 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d81fd3067..240148594 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -138,8 +138,6 @@ USEDTOFAIL=" " ROUND2=" -15 -35 664 665 667 @@ -153,6 +151,8 @@ NOWSUCCESS=" 28 65 566 +15 +35 " # 470..663 from Robin @@ -186,7 +186,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - scripts/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 088205b22..d2a7aac5f 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -69,7 +69,7 @@ echo "Testing prepared witnesses.." for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - zero_bin/tools/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then From c000b5a55d7bccdeed29da60559533114279aa77 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 22:46:57 +0200 Subject: [PATCH 17/53] remove redtape for JUMP/I --- .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/prover_input.rs | 6 +- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 5 +- zero/src/rpc/jerigon.rs | 11 ++-- zero/src/rpc/jumpdest.rs | 58 ++++++++++--------- zero/src/rpc/native/txn.rs | 2 +- 7 files changed, 45 insertions(+), 41 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index aae20ae78..f4fb947e6 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -26,9 +26,9 @@ mod signed_syscalls; mod transaction_parsing; mod transient_storage; -use ::core::iter::Iterator as _; use std::{ops::Range, str::FromStr}; +use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e2ea4b179..c561504df 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,10 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - info!("SIMW {:#?}", sim); - info!("RPCW {:#?}", rpc); + info!("SIM {:#?}", sim); + info!("RPC {:#?}", rpc); - self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 80515c9e1..dc59806b9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 240148594..baee31ea7 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,7 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace -export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` @@ -142,10 +141,10 @@ ROUND2=" 665 667 670 -444 " NOWSUCCESS=" +444 4 5 28 diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 9a9946a23..ab8cf8d37 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -16,9 +16,12 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; -use tracing::debug; +use tracing::info; -use super::{fetch_other_block_data, jumpdest::{self, get_normalized_structlog}}; +use super::{ + fetch_other_block_data, + jumpdest::{self, get_normalized_structlog}, +}; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; /// Transaction traces retrieved from Erigon zeroTracer. @@ -167,14 +170,14 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!( + info!( "{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error ); None }, |jdt| { - debug!( + info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 4861aeeca..533a9229b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -143,7 +143,7 @@ pub(crate) fn generate_jumpdest_table( // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. - let mut prev_jump = None; + let mut prev_jump: Option = None; // Call depth of the previous `entry`. We initialize to 0 as this compares // smaller to 1. @@ -167,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - // trace!("TX: {:?}", tx.hash); - // trace!("STEP: {:?}", step); - // trace!("STEPS: {:?}", struct_log.len()); - // trace!("OPCODE: {}", entry.op.as_str()); - // trace!("CODE: {:?}", code_hash); - // trace!("CTX: {:?}", ctx); - // trace!("CURR_DEPTH: {:?}", curr_depth); - // trace!("{:#?}\n", entry); + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -226,7 +226,7 @@ pub(crate) fn generate_jumpdest_table( let operands = 3; ensure!( evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); let [_value, _offset, _size, ..] = evm_stack[..] else { @@ -301,18 +301,18 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, ..] = evm_stack[..] else { + let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", - *counter, - u64::MAX - ); - let jump_target: u64 = counter.to(); - - prev_jump = Some(jump_target); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); + + prev_jump = Some(*jump_target); } "JUMPI" => { ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); @@ -324,25 +324,27 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, condition, ..] = evm_stack[..] else { + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." - ); - let jump_target: u64 = counter.to(); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { - Some(jump_target) + Some(*jump_target) } else { None }; } "JUMPDEST" => { let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == entry.pc + jmp_target == U256::from(entry.pc) } else { false }; diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index fdfdf23f5..1305f9010 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,6 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use crate::rpc::Compat; use __compat_primitive_types::{H256, U256}; use alloy::{ primitives::{keccak256, Address, B256}, @@ -26,6 +25,7 @@ use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::rpc::jumpdest::{self, get_normalized_structlog}; +use crate::rpc::Compat; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( From ec817013e670ae78b7fedc95dd28f27de5ca7e81 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 00:12:35 +0200 Subject: [PATCH 18/53] misc --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 8 ++++---- zero/src/rpc/jumpdest.rs | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index dc59806b9..f525d8b4c 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index baee31ea7..fe2eba2fa 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,8 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=67108864 +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -138,9 +138,9 @@ USEDTOFAIL=" ROUND2=" 664 -665 667 670 +665 " NOWSUCCESS=" @@ -185,7 +185,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 533a9229b..d6dc4579d 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,7 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::{instrument, trace}; +use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. @@ -98,7 +98,6 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], From bff471ed894b97df98171abdafbf2f33b8ee835a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 11:49:25 +0200 Subject: [PATCH 19/53] fix ci --- scripts/prove_stdio.sh | 11 +++++------ scripts/test_jerigon.sh | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index f525d8b4c..8c5e60fe9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -30,17 +30,16 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -#export TOKIO_WORKER_THREADS=$num_procs +export TOKIO_WORKER_THREADS=$num_procs -#export RUST_MIN_STACK=33554432 -#export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_MIN_STACK=33554432 +export RUST_BACKTRACE=full +export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index fe2eba2fa..5bb8e7293 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,6 @@ set -uo pipefail -#export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= if [ -z $RPC ]; then # You must set an RPC endpoint From ca9620dc26c01de7a77540323317b30e36fa180f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 16:35:39 +0200 Subject: [PATCH 20/53] minimize diff --- .gitignore | 1 - .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 2 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- .../src/cpu/kernel/tests/mod.rs | 1 - .../src/generation/jumpdest.rs | 16 +------ evm_arithmetization/src/generation/mod.rs | 6 +-- .../src/generation/prover_input.rs | 16 +++---- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- scripts/prove_stdio.sh | 9 ++-- scripts/test_jerigon.sh | 11 +---- trace_decoder/src/core.rs | 18 ++++---- trace_decoder/src/interface.rs | 2 +- trace_decoder/src/jumpdest.rs | 1 - trace_decoder/src/lib.rs | 1 - zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jumpdest.rs | 43 +++++++------------ zero/src/rpc/native/mod.rs | 3 +- zero/src/rpc/native/txn.rs | 15 +++++-- 26 files changed, 71 insertions(+), 98 deletions(-) delete mode 100644 trace_decoder/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index d262be31e..9f5ee7a64 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ # Misc ###### /**/*.ignoreme -**/output.log /**/*.ipynb /**/*.log /**/*.out diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 2b35b2a1c..106608935 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 85d43771f..9c2f7bfac 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -118,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, + pub(crate) jumpdest_table: Option, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, - pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fbae6647d..4a6dc9447 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 673b516b0..716e238d0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index f4fb947e6..2b983d099 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -28,7 +28,6 @@ mod transient_storage; use std::{ops::Range, str::FromStr}; -use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index fc2a8e1a6..9a3201cce 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -51,18 +51,11 @@ impl JumpDestTableWitness { /// Creates the required `ctx` keys and `code_hash`. Idempotent. pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { (*self).entry(code_hash).or_default().insert(ctx, offset); - - // TODO(einar) remove before publishing PR. - assert!(self.0.contains_key(&code_hash)); - assert!(self.0[&code_hash].0.contains_key(&ctx)); - assert!(self.0[&code_hash].0[&ctx].contains(&offset)); } pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { let mut curr_max_ctx = prev_max_ctx; - // TODO: Opportunity for optimization: Simulate to generate only missing - // JUMPDEST tables. for (code_hash, ctx_tbl) in (*other).iter() { for (ctx, jumpdests) in ctx_tbl.0.iter() { let batch_ctx = prev_max_ctx + ctx; @@ -70,12 +63,7 @@ impl JumpDestTableWitness { for offset in jumpdests { self.insert(*code_hash, batch_ctx, *offset); - - assert!(self.0.contains_key(code_hash)); - assert!(self.0[code_hash].0.contains_key(&batch_ctx)); - assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); } - // dbg!(&self); } } @@ -90,7 +78,7 @@ impl JumpDestTableWitness { impl Display for JumpDestTableWitness { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableWitness ===")?; + writeln!(f, "\n=== JumpDestTableWitness ===")?; for (code, ctxtbls) in &self.0 { write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; @@ -115,7 +103,7 @@ impl Display for ContextJumpDests { impl Display for JumpDestTableProcessed { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableProcessed ===")?; + writeln!(f, "\n=== JumpDestTableProcessed ===")?; let v = sorted(self.0.clone()); for (ctx, code) in v { diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index e5ec7bde3..0a775eed5 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -240,7 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), - jumpdest_tables: self.jumpdest_tables.clone(), + jumpdest_table: self.jumpdest_table.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index c561504df..e30ea3b6e 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -43,8 +43,7 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; -/// A set of contract code as a byte arrays. From this a mapping: hash -> -/// contract can be built. +/// A set to hold contract code as a byte vectors. pub type CodeDb = BTreeSet>; /// Prover input function represented as a scoped function name. @@ -768,11 +767,11 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_tables); + dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code - let rpcw = &self.inputs.jumpdest_tables; + let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -809,13 +808,10 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); - trace!( - "ctx: {ctx}, code_hash: {:?} code: {:?}", - keccak(code.clone()), - code - ); + let code_hash = keccak(code.clone()); + trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); for offset in jumpdest_table.clone() { - jdtw.insert(keccak(code.clone()), ctx, offset); + jdtw.insert(code_hash, ctx, offset); } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 12a84fbe7..f3af1539e 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 475d03a8e..d251ed6fc 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 8d30f85d1..576eb1e57 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 951ebbb28..9de1e5d63 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 221113b26..f3e6996c1 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index de8e096cf..933f3fd2c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 2aa776e54..6f4686ef6 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 8c5e60fe9..2b804f98a 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,6 +21,9 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +echo "Block batch size: $BLOCK_BATCH_SIZE" + BATCH_SIZE=${BATCH_SIZE:=1} echo "Batch size: $BATCH_SIZE" @@ -28,15 +31,13 @@ OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${REPO_ROOT}/test.out" -BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' @@ -103,7 +104,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5bb8e7293..6d1125216 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -163,15 +163,10 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" -BLOCKS="$ROUND2" +BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` -#echo "Testing: $BLOCKS" +echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt echo "---------------------------" | tee -a witnesses/jerigon_results.txt @@ -194,5 +189,3 @@ for BLOCK in $BLOCKS; do fi printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done - -exit 0 diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2b1a0e49c..cd410e58b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -77,7 +77,6 @@ pub fn entrypoint( withdrawals, )?; - dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() @@ -117,32 +116,35 @@ pub fn entrypoint( checkpoint_state_trie_root, checkpoint_consolidated_hash, contract_code: { - let cc = contract_code.into_iter(); - let initcodes = byte_code .iter() - .filter_map(|nonempty_txn_raw| -> Option> { + .filter_map(|nonempty_txn_bytes| -> Option> { let tx_envelope = - TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + TxEnvelope::decode(&mut &nonempty_txn_bytes[..]).unwrap(); match tx_envelope.to() { TxKind::Create => Some(tx_envelope.input().to_vec()), TxKind::Call(_address) => None, } }); - cc.chain(initcodes) + contract_code + .into_iter() + .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) .collect() }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables: { + jumpdest_table: { + // Note that this causes any batch containing just a single `None` to collapse + // into a `None`, which causing failover to simulating jumpdest analysis for the + // whole batch. There is an optimization opportunity here. jumpdest_tables .into_iter() .collect::>>() - .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0) }, }, ) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 5c72d6b3f..81cc87bbe 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -170,7 +170,7 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Question: Why has this has been removed upstream. Proably unused. +// Review: Re-adding this. It has been removed upstream. impl ContractCodeUsage { /// Get code hash from a read or write operation of contract code. pub fn get_code_hash(&self) -> H256 { diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs deleted file mode 100644 index 8b1378917..000000000 --- a/trace_decoder/src/jumpdest.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index aaab848d4..8c54eea8e 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -60,7 +60,6 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 -mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index d9f25901a..b9751196c 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -207,7 +207,7 @@ async fn main() -> anyhow::Result<()> { .with( tracing_subscriber::fmt::layer() // With the default configuration trace information is written - // to stdout, but we already use to write our payload (the witness). + // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) .with_ansi(false) .compact() diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d6dc4579d..a48d4759a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -65,11 +65,12 @@ where TransportT: Transport + Clone, { // Optimization: It may be a better default to pull the stack immediately. - let light_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) .await?; - let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + let structlogs_opt: Option> = + normalize_structlog(stackonly_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); trace!("Need structlog with memory: {need_memory}"); @@ -107,7 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); - // This does not contain `initcodes`. + // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -192,22 +193,24 @@ pub(crate) fn generate_jumpdest_table( }; let callee_address = { - // Clear the upper half of the operand. let callee_raw = *address; - // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); - // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); - ensure!(callee_raw <= U256::from(U160::MAX)); let lower_20_bytes = U160::from(callee_raw); Address::from(lower_20_bytes) }; + if callee_addr_to_code_hash.contains_key(&callee_address) { + let next_code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((next_code_hash, next_ctx_available)); + }; + if precompiles().contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); - } else if callee_addr_to_code_hash.contains_key(&callee_address) { - let code_hash = callee_addr_to_code_hash[&callee_address]; - call_stack.push((code_hash, next_ctx_available)); - } else { + }; + + if callee_addr_to_code_hash.contains_key(&callee_address).not() + && precompiles().contains(&callee_address).not() + { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun trace!( @@ -303,13 +306,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); prev_jump = Some(*jump_target); } @@ -326,13 +322,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { @@ -347,7 +336,7 @@ pub(crate) fn generate_jumpdest_table( } else { false }; - let jumpdest_offset = entry.pc as usize; + let jumpdest_offset = TryInto::::try_into(entry.pc)?; if jumped_here { jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 11766b148..5563cad5b 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -15,8 +15,7 @@ use crate::provider::CachedProvider; mod state; mod txn; -pub use txn::process_transaction; -pub use txn::process_transactions; +pub use txn::{process_transaction, process_transactions}; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 1305f9010..34ed820df 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -22,7 +22,7 @@ use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::debug; +use tracing::info; use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; @@ -86,10 +86,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + info!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - Some, + |jdt| { + info!( + "{:#?}: JumpDestTable generation succeeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); From 4c97c0f8537aca0b279933286b13100890ac8a57 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 17:14:19 +0200 Subject: [PATCH 21/53] include whole function in timeout --- zero/src/rpc/jumpdest.rs | 48 +++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index a48d4759a..f44aa7a43 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: Vec) -> bool { +fn trace_contains_create2(structlog: &[StructLog]) -> bool { structlog.iter().any(|entry| entry.op == "CREATE2") } @@ -64,27 +64,39 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - // Optimization: It may be a better default to pull the stack immediately. - let stackonly_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) - .await?; + let inner = async { + // Optimization: It may be a better default to pull the stack immediately. + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) + .await?; - let structlogs_opt: Option> = - normalize_structlog(stackonly_structlog_trace).await; + let stackonly_structlog_opt: Option> = + normalize_structlog(&stackonly_structlog_trace).await; - let need_memory = structlogs_opt.is_some_and(trace_contains_create2); - trace!("Need structlog with memory: {need_memory}"); + let need_memory = stackonly_structlog_opt + .as_deref() + .is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider.debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ); + if need_memory.not() { + return Ok(stackonly_structlog_opt); + }; + + let memory_structlog_fut = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); + + let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + + Ok::>, RpcError>(memory_structlog) + }; - match timeout(TIMEOUT_LIMIT, structlog).await { + match timeout(TIMEOUT_LIMIT, inner).await { Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( ellapsed_error, )))), - Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + Ok(structlog_res) => Ok(structlog_res?), } } @@ -470,11 +482,11 @@ pub mod structlogprime { } pub(crate) async fn normalize_structlog( - unnormalized_structlog: GethTrace, + unnormalized_structlog: &GethTrace, ) -> Option> { match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) .ok() .map(|s| s.struct_logs), _ => None, From 8bce013d66407b8b097274328dbd0998493d9977 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 18:54:58 +0200 Subject: [PATCH 22/53] avoid ensure macro --- zero/src/rpc/jumpdest.rs | 181 +++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 82 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index f44aa7a43..d916ba4b5 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -176,7 +176,10 @@ pub(crate) fn generate_jumpdest_table( call_stack.pop(); } - ensure!(call_stack.is_empty().not(), "Call stack was empty."); + ensure!( + call_stack.is_empty().not(), + "Call stack was unexpectedly empty." + ); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -190,26 +193,32 @@ pub(crate) fn generate_jumpdest_table( match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 2; // actually 6 or 7. - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + let operands_used = 2; // actually 6 or 7. + + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // maybe increment ctx here + continue; + } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 let [_gas, address, ..] = evm_stack[..] else { unreachable!() }; - let callee_address = { - let callee_raw = *address; - ensure!(callee_raw <= U256::from(U160::MAX)); - let lower_20_bytes = U160::from(callee_raw); - Address::from(lower_20_bytes) + if *address > U256::from(U160::MAX) { + trace!( + "{op}: Callee address {} was larger than possible {}.", + *address, + U256::from(U160::MAX) + ); + continue; }; + let lower_20_bytes = U160::from(*address); + let callee_address = Address::from(lower_20_bytes); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; @@ -231,69 +240,63 @@ pub(crate) fn generate_jumpdest_table( ); } next_ctx_available += 1; - prev_jump = None; } - "CREATE" => { + "CREATE" | "CREATE2" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 3; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, _offset, _size, ..] = evm_stack[..] else { - unreachable!() - }; + let operands_used = 3; - let contract_address = tx.from.create(tx.nonce); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + }; - next_ctx_available += 1; - prev_jump = None; - } - "CREATE2" => { - ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); - // We reverse the stack, so the order matches our assembly code. - let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 4; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, offset, size, _salt, ..] = evm_stack[..] else { + let [_value, offset, size, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset <= U256::from(usize::MAX)); + if *offset > U256::from(usize::MAX) { + trace!( + "{op}: Offset {offset} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let offset: usize = offset.to(); - ensure!(*size <= U256::from(usize::MAX)); + if *size > U256::from(usize::MAX) { + trace!( + "{op}: Size {size} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - // let salt: Word = salt.to_be_bytes(); - ensure!( - entry.memory.is_some() && offset + size <= memory_size, - "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size - ); + if entry.memory.is_none() || offset + size > memory_size { + trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); + continue; + } let memory_raw: &[String] = entry.memory.as_ref().unwrap(); let memory_parsed: Vec> = memory_raw .iter() - .map(|s| { - // let c = s.parse(); - let c = U256::from_str_radix(s, 16); - ensure!(c.is_ok(), "Parsing memory failed."); - let a: U256 = c.unwrap(); - let d: Word = a.to_be_bytes(); - Ok(d) + .map(|mem_line| { + let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; + Ok(mem_line_parsed.to_be_bytes()) }) .collect(); let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); - let memory: Vec = mem_res?.concat(); + if mem_res.is_err() { + trace!( + "{op}: Parsing memory failed with error: {}", + mem_res.unwrap_err() + ); + continue; + } + let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); @@ -303,18 +306,17 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; - prev_jump = None; } "JUMP" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 1; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + } let [jump_target, ..] = evm_stack[..] else { unreachable!() }; @@ -322,42 +324,55 @@ pub(crate) fn generate_jumpdest_table( prev_jump = Some(*jump_target); } "JUMPI" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 2; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len()); + continue; + }; + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; let jump_condition = condition.is_zero().not(); - prev_jump = if jump_condition { - Some(*jump_target) - } else { - None - }; + if jump_condition { + prev_jump = Some(*jump_target) + } } "JUMPDEST" => { - let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == U256::from(entry.pc) - } else { - false - }; - let jumpdest_offset = TryInto::::try_into(entry.pc)?; - if jumped_here { - jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + let mut jumped_here = false; + + if let Some(jmp_target) = prev_jump { + jumped_here = jmp_target == U256::from(entry.pc); } - // else: we do not care about JUMPDESTs reached through fall-through. prev_jump = None; + + if jumped_here.not() { + trace!( + "{op}: JUMPDESTs at offset {} was reached through fall-through.", + entry.pc + ); + continue; + } + + let jumpdest_offset = TryInto::::try_into(entry.pc); + if jumpdest_offset.is_err() { + trace!( + "{op}: Could not cast offset {} to usize {}.", + entry.pc, + usize::MAX + ); + continue; + } + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { - next_ctx_available += 1; prev_jump = None; + next_ctx_available += 1; } _ => { prev_jump = None; @@ -367,6 +382,8 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } +/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; From b0ebc2c2fe0999940ed52f105a4df0c43d150232 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 19:25:33 +0200 Subject: [PATCH 23/53] fix CREATE --- zero/src/rpc/jumpdest.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d916ba4b5..8ef51bbfa 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -52,7 +52,9 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } fn trace_contains_create2(structlog: &[StructLog]) -> bool { - structlog.iter().any(|entry| entry.op == "CREATE2") + structlog + .iter() + .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } // Gets the lightest possible structlog for transcation `tx_hash`. From 62c7053d36a16a67287a70fb370773cb66a091e2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 11:27:07 +0200 Subject: [PATCH 24/53] small adjustments --- Cargo.lock | 1 - Cargo.toml | 2 + evm_arithmetization/Cargo.toml | 1 - .../src/generation/prover_input.rs | 2 +- scripts/test_native.sh | 51 ++++++++++++------- zero/src/rpc/jumpdest.rs | 6 +-- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f929a38c1..0ed81ea1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2068,7 +2068,6 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", - "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", diff --git a/Cargo.toml b/Cargo.toml index 386ecff5e..cac073af2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" +enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -89,6 +90,7 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" +serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index e8e832c2d..ffdb8d2f5 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,7 +15,6 @@ homepage.workspace = true keywords.workspace = true [dependencies] -__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e30ea3b6e..b502b68af 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,7 +767,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code diff --git a/scripts/test_native.sh b/scripts/test_native.sh index d2a7aac5f..943a00ca3 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,22 +10,25 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export RAYON_NUM_THREADS=1 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +#export RUST_LOG=info +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=33554432 -MAINNETBLOCKS=" +CANCUNBLOCKS=" 20548415 20240058 19665756 20634472 19807080 20634403 +" + +PRECANCUN=" 19096840 19240700 " @@ -33,17 +36,18 @@ MAINNETBLOCKS=" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 -NUMRANDOMBLOCKS=10 +NUMRANDOMBLOCKS=100 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" -BLOCKS="$MAINNETBLOCKS" +BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" echo "Downloading witnesses.." +echo "------------------------"| tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" @@ -56,20 +60,15 @@ for BLOCK in $BLOCKS; do then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break + else + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt fi - - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt done echo "Witness for block $BLOCK ($WITNESS) prepared." -done - -echo "Finished downloading witnesses." -echo "Testing prepared witnesses.." -for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then @@ -80,4 +79,20 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt done -echo "Finished testing witnesses." +#echo "Finished downloading witnesses." +#echo "Testing prepared witnesses.." +# +#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do +# echo "Testing $WITNESS" +# ./prove_stdio.sh $WITNESS test_only +# EXITCODE=$? +# if [ $EXITCODE -eq 0 ] +# then +# RESULT="success" +# else +# RESULT="failure" +# fi +# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +#done +# +#echo "Finished testing witnesses." diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8ef51bbfa..38894679a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: &[StructLog]) -> bool { +fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") @@ -77,7 +77,7 @@ where let need_memory = stackonly_structlog_opt .as_deref() - .is_some_and(trace_contains_create2); + .is_some_and(trace_contains_create); trace!("Need structlog with memory: {need_memory}"); if need_memory.not() { From 74b86fd916f6ab41fbec9cbe4e69dc16448be3de Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 13:39:22 +0200 Subject: [PATCH 25/53] fmt --- zero/src/rpc/jumpdest.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 38894679a..d9ba8cd4c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,13 +51,15 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Predicate that determines whether a `StructLog` that includes memory is +/// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } -// Gets the lightest possible structlog for transcation `tx_hash`. +/// Gets the lightest possible structlog for transcation `tx_hash`. pub(crate) async fn get_normalized_structlog( provider: &ProviderT, tx_hash: &B256, From c8be8888fb190cdcfa5abcaff04f0bfbf575684b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 14:29:46 +0200 Subject: [PATCH 26/53] feedback --- scripts/test_native.sh | 2 +- zero/src/prover/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 943a00ca3..56de59a49 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail export RPC= if [ -z $RPC ]; then diff --git a/zero/src/prover/cli.rs b/zero/src/prover/cli.rs index 0dfa383b6..a6cdaebf9 100644 --- a/zero/src/prover/cli.rs +++ b/zero/src/prover/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] From d00439fc64fa2c9eb88aa58228ea007bf4e03e53 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 12:20:12 +0200 Subject: [PATCH 27/53] feedback --- .gitignore | 1 - zero/src/rpc/jumpdest.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9f5ee7a64..0016ff3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,3 @@ ################# /proofs /**/*.zkproof -/witnesses diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d9ba8cd4c..558804bc7 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -386,7 +386,7 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } -/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// This module exists as a workaround for parsing `StructLog`. The `error` /// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; From 6876c07e9b1f07b7afb2e673288834fc093e6355 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 16:31:51 +0200 Subject: [PATCH 28/53] Add JumpdestSrc parameter --- trace_decoder/src/core.rs | 4 ++-- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/mod.rs | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index cd410e58b..78c9d9b96 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -6,10 +6,9 @@ use std::{ mem, }; -use alloy::primitives::address; use alloy::{ consensus::{Transaction, TxEnvelope}, - primitives::TxKind, + primitives::{address, TxKind}, rlp::Decodable as _, }; use alloy_compat::Compat as _; @@ -138,6 +137,7 @@ pub fn entrypoint( block_hashes: b_hashes.clone(), burn_addr, jumpdest_table: { + // TODO See the issue Simulate to get jumpdests on a per-transaction basis #653. // Note that this causes any batch containing just a single `None` to collapse // into a `None`, which causing failover to simulating jumpdest analysis for the // whole batch. There is an optimization opportunity here. diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index b9751196c..115a7619e 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -15,6 +15,7 @@ use zero::block_interval::BlockIntervalStream; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; +use zero::rpc::JumpdestSrc; use self::rpc::{retry::build_http_retry_provider, RpcType}; @@ -34,6 +35,9 @@ struct RpcToolConfig { /// The RPC Tracer Type. #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] backoff: u64, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e3c218eb6..da1ac474b 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -39,6 +39,14 @@ pub enum RpcType { Native, } +/// The Jumpdest source type. +#[derive(ValueEnum, Clone, Debug, Copy)] +pub enum JumpdestSrc { + Simulation, + Zero, + Jerigon, +} + /// Obtain the prover input for one block pub async fn block_prover_input( cached_provider: Arc>, From 60efef9383b45954c9a735e5a9ae669fa89d43e7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 17:36:09 +0200 Subject: [PATCH 29/53] Refactor --- zero/src/bin/leader.rs | 2 ++ zero/src/bin/leader/cli.rs | 5 ++++- zero/src/bin/leader/client.rs | 4 +++- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/jerigon.rs | 36 ++++++++++++++++++++--------------- zero/src/rpc/mod.rs | 17 +++++++++++++++-- zero/src/rpc/native/mod.rs | 3 +++ 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 728f6bb0e..775439fde 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -72,6 +72,7 @@ async fn main() -> Result<()> { Command::Rpc { rpc_url, rpc_type, + jumpdest_src, block_interval, checkpoint_block_number, previous_proof, @@ -91,6 +92,7 @@ async fn main() -> Result<()> { backoff, max_retries, block_time, + jumpdest_src, }, block_interval, LeaderConfig { diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 3569efc41..78ad0fbdb 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -4,7 +4,7 @@ use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueHint}; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; -use zero::rpc::RpcType; +use zero::rpc::{JumpdestSrc, RpcType}; /// zero-bin leader config #[derive(Parser)] @@ -43,6 +43,9 @@ pub(crate) enum Command { // The node RPC type (jerigon / native). #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] block_interval: String, diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index 6825b6683..136fb0845 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -10,7 +10,7 @@ use tracing::info; use zero::block_interval::{BlockInterval, BlockIntervalStream}; use zero::pre_checks::check_previous_proof_and_checkpoint; use zero::prover::{self, BlockProverInput, ProverConfig}; -use zero::rpc; +use zero::rpc::{self, JumpdestSrc}; use zero::rpc::{retry::build_http_retry_provider, RpcType}; #[derive(Debug)] @@ -20,6 +20,7 @@ pub struct RpcParams { pub backoff: u64, pub max_retries: u32, pub block_time: u64, + pub jumpdest_src: JumpdestSrc, } #[derive(Debug)] @@ -87,6 +88,7 @@ pub(crate) async fn client_main( block_id, leader_config.checkpoint_block_number, rpc_params.rpc_type, + rpc_params.jumpdest_src, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 115a7619e..3c5c3ed88 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -25,6 +25,7 @@ struct FetchParams { pub end_block: u64, pub checkpoint_block_number: Option, pub rpc_type: RpcType, + pub jumpdest_src: JumpdestSrc, } #[derive(Args, Clone, Debug)] @@ -105,6 +106,7 @@ where block_id, checkpoint_block_number, params.rpc_type, + params.jumpdest_src, ) .await?; @@ -133,6 +135,7 @@ impl Cli { end_block, checkpoint_block_number, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = @@ -159,6 +162,7 @@ impl Cli { end_block: block_number, checkpoint_block_number: None, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index ab8cf8d37..15ad0edf1 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,6 +1,8 @@ +use core::iter::Iterator; use std::collections::BTreeMap; use std::ops::Deref as _; +use __compat_primitive_types::H160; use alloy::{ providers::Provider, rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, @@ -21,6 +23,7 @@ use tracing::info; use super::{ fetch_other_block_data, jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, }; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; @@ -36,6 +39,7 @@ pub async fn block_prover_input( cached_provider: std::sync::Arc>, target_block_id: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -65,17 +69,18 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results - .iter() - .map(|TxnInfo { traces, meta: _ }| traces) - .collect::>(); - - let jdts: Vec> = process_transactions( - &block, - cached_provider.get_provider().await?.deref(), - &tx_traces, - ) - .await?; + let jdts: Vec> = match jumpdest_src { + JumpdestSrc::Simulation => vec![None; tx_results.len()], + JumpdestSrc::Zero => { + process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + tx_results.iter().map(|TxnInfo { traces, meta: _ }| traces), // &tx_traces, + ) + .await? + } + JumpdestSrc::Jerigon => todo!(), + }; // weave in the JDTs let txn_info = tx_results @@ -124,14 +129,15 @@ where } /// Processes the transactions in the given block and updates the code db. -pub async fn process_transactions( +pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, - tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], + tx_traces: I, ) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, + I: Iterator>, { let futures = block .transactions @@ -139,7 +145,7 @@ where .context("No transactions in block")? .iter() .zip(tx_traces) - .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); let vec_of_res = futures.collect::>().await; @@ -151,7 +157,7 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, - tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, + tx_trace: &BTreeMap, ) -> anyhow::Result> where ProviderT: Provider, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index da1ac474b..061cd9743 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -53,6 +53,7 @@ pub async fn block_prover_input( block_id: BlockId, checkpoint_block_number: u64, rpc_type: RpcType, + jumpdest_src: JumpdestSrc, ) -> Result where ProviderT: Provider, @@ -60,10 +61,22 @@ where { match rpc_type { RpcType::Jerigon => { - jerigon::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + jerigon::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } RpcType::Native => { - native::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + native::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } } } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 5563cad5b..6dcf07af3 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -17,11 +17,14 @@ mod txn; pub use txn::{process_transaction, process_transactions}; +use super::JumpdestSrc; + /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( provider: Arc>, block_number: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, From b07752dd60a45b626c0fc168661ba0cee76640e9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 18:12:43 +0200 Subject: [PATCH 30/53] Add jmp src to native --- zero/src/rpc/native/mod.rs | 11 ++++++++--- zero/src/rpc/native/txn.rs | 39 +++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 6dcf07af3..76dd302ae 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -31,7 +31,7 @@ where TransportT: Transport + Clone, { let (block_trace, other_data) = try_join!( - process_block_trace(provider.clone(), block_number), + process_block_trace(provider.clone(), block_number, jumpdest_src), crate::rpc::fetch_other_block_data(provider.clone(), block_number, checkpoint_block_number) )?; @@ -45,6 +45,7 @@ where pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -54,8 +55,12 @@ where .get_block(block_number, BlockTransactionsKind::Full) .await?; - let (code_db, txn_info) = - txn::process_transactions(&block, cached_provider.get_provider().await?.deref()).await?; + let (code_db, txn_info) = txn::process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + jumpdest_src, + ) + .await?; let trie_pre_images = state::process_state_witness(cached_provider, block, &txn_info).await?; Ok(BlockTrace { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 34ed820df..582b7ee8f 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use __compat_primitive_types::{H256, U256}; @@ -24,13 +25,17 @@ use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::info; -use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; +use crate::rpc::{ + jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, +}; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( block: &Block, provider: &ProviderT, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, Vec)> where ProviderT: Provider, @@ -41,7 +46,7 @@ where .as_transactions() .context("No transactions in block")? .iter() - .map(|tx| process_transaction(provider, tx)) + .map(|tx| process_transaction(provider, tx, jumpdest_src)) .collect::>() .try_fold( (BTreeSet::new(), Vec::new()), @@ -59,13 +64,14 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, TxnInfo)> where ProviderT: Provider, TransportT: Transport + Clone, { let (tx_receipt, pre_trace, diff_trace, structlog_opt) = - fetch_tx_data(provider, &tx.hash).await?; + fetch_tx_data(provider, &tx.hash, jumpdest_src).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); @@ -125,6 +131,7 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<( ::ReceiptResponse, GethTrace, @@ -138,14 +145,24 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); - - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( - tx_receipt_fut, - pre_trace_fut, - diff_trace_fut, - structlog_trace_fut, - )?; + + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { + JumpdestSrc::Zero => { + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); + futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut, + )? + } + JumpdestSrc::Simulation => { + let (tx_receipt, pre_trace, diff_trace) = + futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + (tx_receipt, pre_trace, diff_trace, None) + } + JumpdestSrc::Jerigon => todo!(), + }; Ok(( tx_receipt.context("Transaction receipt not found.")?, From 66ea81151d7b12d4dfedb846eb3d2e0db560ef16 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:06:22 +0200 Subject: [PATCH 31/53] Feedback --- trace_decoder/src/core.rs | 9 +++++---- trace_decoder/src/lib.rs | 1 + zero/src/rpc/jumpdest.rs | 14 ++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 78c9d9b96..4890a68b4 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -34,6 +34,11 @@ use crate::{ SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, }; +/// Addresses of precompiled Ethereum contracts. +pub const PRECOMPILE_ADDRESSES: Range = + address!("0000000000000000000000000000000000000001") + ..address!("000000000000000000000000000000000000000a"); + /// TODO(0xaatif): document this after https://github.com/0xPolygonZero/zk_evm/issues/275 pub fn entrypoint( trace: BlockTrace, @@ -496,10 +501,6 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } else { // Simple state access - const PRECOMPILE_ADDRESSES: Range = - address!("0000000000000000000000000000000000000001") - ..address!("000000000000000000000000000000000000000a"); - if receipt.status || !PRECOMPILE_ADDRESSES.contains(&addr.compat()) { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/pull/613 // masking like this SHOULD be a space-saving optimization, diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index 8c54eea8e..587707883 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -67,6 +67,7 @@ mod typed_mpt; mod wire; pub use core::entrypoint; +pub use core::PRECOMPILE_ADDRESSES; mod core; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 558804bc7..fc42ebb11 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,6 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; +use trace_decoder::PRECOMPILE_ADDRESSES; use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over @@ -145,7 +146,9 @@ pub(crate) fn generate_jumpdest_table( ); let entrypoint_code_hash: H256 = match tx.to { - Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { + return Ok(jumpdest_table) + } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { return Ok(jumpdest_table) } @@ -201,11 +204,13 @@ pub(crate) fn generate_jumpdest_table( ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands_used = 2; // actually 6 or 7. + // These opcodes expect 6 or 7 operands on the stack, but for jumpdest-table + // generation we only use 2, and failures will be handled in + // next iteration by popping the stack accordingly. + let operands_used = 2; if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); - // maybe increment ctx here continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -304,9 +309,6 @@ pub(crate) fn generate_jumpdest_table( let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); - // let contract_address = tx.from.create2_from_code(salt, init_code); - // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - // let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; From f230b84423df8f268d4d56a98bc2e4042bf7649d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:16:24 +0200 Subject: [PATCH 32/53] fixup! Feedback --- zero/src/rpc/jumpdest.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index fc42ebb11..238ac7b7a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -3,9 +3,7 @@ use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::HashSet; use std::ops::Not as _; -use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; @@ -105,15 +103,6 @@ where } } -/// Provides a way to check in constant time if an address points to a -/// precompile. -fn precompiles() -> &'static HashSet
{ - static PRECOMPILES: OnceLock> = OnceLock::new(); - PRECOMPILES.get_or_init(|| { - HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) - }) -} - /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. pub(crate) fn generate_jumpdest_table( @@ -234,12 +223,12 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((next_code_hash, next_ctx_available)); }; - if precompiles().contains(&callee_address) { + if PRECOMPILE_ADDRESSES.contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); }; if callee_addr_to_code_hash.contains_key(&callee_address).not() - && precompiles().contains(&callee_address).not() + && PRECOMPILE_ADDRESSES.contains(&callee_address).not() { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun From 90722a30578b7e3aac2d5b107fae46ebb79e6f3c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:29:58 +0200 Subject: [PATCH 33/53] feedback --- zero/src/rpc/jerigon.rs | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 15ad0edf1..5fcf7f584 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -79,36 +79,21 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!(), + JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), }; + let mut code_db = CodeDb::default(); // weave in the JDTs let txn_info = tx_results .into_iter() .zip(jdts) - .map( - |( - TxnInfo { - traces, - meta: - TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: _, - }, - }, - jdt, - )| TxnInfo { - traces, - meta: TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: jdt, - }, - }, - ) + .map(|(mut tx_info, jdt)| { + tx_info.meta.jumpdest_table = jdt.map(|(j, c)| { + code_db.extend(c); + j + }); + tx_info + }) .collect(); let other_data = @@ -148,8 +133,11 @@ where .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); - let vec_of_res = futures.collect::>().await; - vec_of_res.into_iter().collect::, _>>() + futures + .collect::>() + .await + .into_iter() + .collect::, _>>() } /// Processes the transaction with the given transaction hash and updates the From a0e0879d74624adb1c0bab36e62c2d78bc34da0e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:33:07 +0200 Subject: [PATCH 34/53] fix missing code for CREATE --- .../src/cpu/kernel/interpreter.rs | 5 +- .../src/generation/prover_input.rs | 11 +-- scripts/test_jerigon.sh | 86 ++++++++++++++++++- trace_decoder/src/core.rs | 22 ++++- zero/src/rpc/jerigon.rs | 25 +++--- zero/src/rpc/jumpdest.rs | 11 ++- zero/src/rpc/native/txn.rs | 18 ++-- 7 files changed, 143 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 9c2f7bfac..5ab8fc659 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -173,9 +173,10 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( &vec![] }; trace!( - "code: {:?}, code_addr: {:?} <============", + "code: {:?}, code_addr: {:?}, {:?} <============", &code, - &code_addr + &code_addr, + code_map.contains_key(code_addr), ); trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b502b68af..b7d4a6788 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,23 +767,20 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // w for witness - // let txn_idx = self.next_txn_index - 1; - // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + dbg!(&self.inputs.jumpdest_table); + dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables part2"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { + if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); assert_eq!(rw, &sw); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 6d1125216..5426b5b01 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -153,6 +153,89 @@ NOWSUCCESS=" 35 " +ROUND3=" +125 +127 +131 +132 +136 +141 +142 +143 +145 +149 +150 +151 +153 +154 +186 +187 +188 +190 +193 +195 +197 +199 +201 +214 +220 +221 +222 +223 +226 +228 +229 +230 +231 +232 +234 +242 +256 +257 +258 +262 +264 +267 +268 +282 +284 +285 +287 +292 +294 +295 +301 +303 +304 +321 +325 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +566 +570 +664 +77 +548 +" + + # 470..663 from Robin for i in {470..663} do @@ -163,7 +246,8 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" +#$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" +BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 4890a68b4..c6e530a75 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -132,11 +132,24 @@ pub fn entrypoint( } }); - contract_code + let initmap: HashMap<_, _> = initcodes .into_iter() - .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) - .collect() + .collect(); + + let contractmap: HashMap<_, _> = contract_code + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + + log::trace!("Initmap {:?}", initmap); + log::trace!("Contractmap {:?}", contractmap); + //log::trace!("DECODER: {:#?}", res); + + let mut c = code.inner.clone(); + c.extend(contractmap); + c.extend(initmap); + c }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -643,9 +656,10 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. +#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - inner: HashMap>, + pub inner: HashMap>, } impl Hash2Code { diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5fcf7f584..0b79cfd70 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -10,14 +10,12 @@ use alloy::{ }; use alloy_primitives::Address; use anyhow::Context as _; -use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::FuturesOrdered; use futures::StreamExt as _; use serde::Deserialize; use serde_json::json; -use trace_decoder::{ - BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, -}; +use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnTrace}; use tracing::info; use super::{ @@ -69,7 +67,7 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let jdts: Vec> = match jumpdest_src { + let jdts: Vec> = match jumpdest_src { JumpdestSrc::Simulation => vec![None; tx_results.len()], JumpdestSrc::Zero => { process_transactions( @@ -107,7 +105,7 @@ where .context("invalid hex returned from call to eth_getWitness")?, }), txn_info, - code_db: Default::default(), + code_db, }, other_data, }) @@ -118,7 +116,7 @@ pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, tx_traces: I, -) -> anyhow::Result>> +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, @@ -146,7 +144,7 @@ pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, tx_trace: &BTreeMap, -) -> anyhow::Result> +) -> anyhow::Result> where ProviderT: Provider, TransportT: Transport + Clone, @@ -161,7 +159,7 @@ where .ok() .flatten(); - let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { info!( @@ -170,15 +168,18 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); - Ok(jumpdest_table) + // TODO + // let jumpdest_table = jc.map(|(j, c)| j); + + Ok(jc) } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 238ac7b7a..ac7173cbf 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -20,6 +20,7 @@ use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::CodeDb; use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; @@ -109,10 +110,11 @@ pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, -) -> anyhow::Result { +) -> anyhow::Result<(JumpDestTableWitness, CodeDb)> { trace!("Generating JUMPDEST table for tx: {}", tx.hash); let mut jumpdest_table = JumpDestTableWitness::default(); + let mut code_db = CodeDb::default(); // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces @@ -136,10 +138,10 @@ pub(crate) fn generate_jumpdest_table( let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) => callee_addr_to_code_hash[&to_address], None => { @@ -297,6 +299,7 @@ pub(crate) fn generate_jumpdest_table( let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; + code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); call_stack.push((init_code_hash, next_ctx_available)); @@ -374,7 +377,7 @@ pub(crate) fn generate_jumpdest_table( } } } - Ok(jumpdest_table) + Ok((jumpdest_table, code_db)) } /// This module exists as a workaround for parsing `StructLog`. The `error` diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 582b7ee8f..c8dab028c 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -76,11 +76,14 @@ where let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { + let (mut code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), GethTrace::PreStateTracer(PreStateFrame::Diff(diff)), - ) => process_tx_traces(access_list, read, diff).await?, + ) => { + info!("{:?} {:?} {:?}", tx.hash, read, diff); + process_tx_traces(access_list, read, diff).await? + } _ => unreachable!(), }; @@ -89,7 +92,7 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { info!( @@ -98,16 +101,21 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); + let jumpdest_table = jc.map(|(j, c)| { + code_db.extend(c); + j + }); + let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From 6bff4e4eac04fec5b881bd75bf9d7921011c0353 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 17:24:44 +0200 Subject: [PATCH 35/53] fix --- scripts/prove_stdio.sh | 4 +-- scripts/test_jerigon.sh | 19 ++++++----- scripts/test_native.sh | 66 ++++++++++++++------------------------- trace_decoder/src/core.rs | 25 +++++++++------ 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 2b804f98a..25db16e19 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -35,9 +35,9 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 +#export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5426b5b01..e30300787 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -9,9 +9,11 @@ if [ -z $RPC ]; then fi mkdir -p witnesses -export RAYON_NUM_THREADS=4 +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=1 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full +#export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' #export RUST_MIN_STACK=67108864 @@ -251,8 +253,8 @@ BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -echo "---------------------------" | tee -a witnesses/jerigon_results.txt +printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` @@ -263,13 +265,16 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - ./prove_stdio.sh $WITNESS test_only + SECONDS=0 + timeout 10m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/jerigon_results.txt done diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 56de59a49..63da02115 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -33,6 +33,11 @@ PRECANCUN=" 19240700 " +#It's visible with block 20727641 +ROUND1=`echo {20727640..20727650}` + + + CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 @@ -42,57 +47,34 @@ RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +BLOCKS="$ROUND1" #BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" -echo "Downloading witnesses.." -echo "------------------------"| tee -a witnesses/native_results.txt +echo "Testing: $BLOCKS" +printf "githash block verdict duration\n" | tee -a witnesses/native_results.txt +echo "------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do + GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" - until [ -f $WITNESS -a -s $WITNESS ]; do - echo "Fetching block $BLOCK" - cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - EXITCODE=$? - - if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] - then - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt - break - else - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt - fi - done - - echo "Witness for block $BLOCK ($WITNESS) prepared." - - echo "Testing $WITNESS" - ./prove_stdio.sh $WITNESS test_only + echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." + export RUST_LOG=info + SECONDS=0 + timeout 30m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/native_results.txt done - -#echo "Finished downloading witnesses." -#echo "Testing prepared witnesses.." -# -#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do -# echo "Testing $WITNESS" -# ./prove_stdio.sh $WITNESS test_only -# EXITCODE=$? -# if [ $EXITCODE -eq 0 ] -# then -# RESULT="success" -# else -# RESULT="failure" -# fi -# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt -#done -# -#echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index c6e530a75..6a32046d3 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -53,7 +53,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db); + code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -132,24 +132,30 @@ pub fn entrypoint( } }); + // TODO convert to Hash2Code let initmap: HashMap<_, _> = initcodes .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); + log::trace!("Initmap {:?}", initmap); let contractmap: HashMap<_, _> = contract_code .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); - - log::trace!("Initmap {:?}", initmap); log::trace!("Contractmap {:?}", contractmap); - //log::trace!("DECODER: {:#?}", res); - let mut c = code.inner.clone(); - c.extend(contractmap); - c.extend(initmap); - c + let codemap: HashMap<_, _> = code_db + .clone() + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + log::trace!("Codemap {:?}", codemap); + + let mut res = codemap; + res.extend(contractmap); + res.extend(initmap); + res }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -656,10 +662,9 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. -#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - pub inner: HashMap>, + inner: HashMap>, } impl Hash2Code { From c26f4752921db7c7f0e14255e1299490a52de4d7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 20:19:56 +0200 Subject: [PATCH 36/53] fix arguments --- evm_arithmetization/src/generation/prover_input.rs | 2 +- scripts/test_jerigon.sh | 2 +- zero/src/bin/leader/cli.rs | 2 +- zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jerigon.rs | 7 ++++--- zero/src/rpc/mod.rs | 7 ++++--- zero/src/rpc/native/txn.rs | 6 +++--- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 43511394a..9015ff4a3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -812,7 +812,7 @@ impl GenerationState { if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); - assert_eq!(rw, &sw); + // assert_eq!(rw, &sw); } info!("SIM {:#?}", sim); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index e30300787..685939540 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -261,7 +261,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 78ad0fbdb..d89a349ea 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,7 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index a55073342..fe29247ee 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,7 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 0b79cfd70..73280fc99 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -68,8 +68,8 @@ where .await?; let jdts: Vec> = match jumpdest_src { - JumpdestSrc::Simulation => vec![None; tx_results.len()], - JumpdestSrc::Zero => { + JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], + JumpdestSrc::ClientFetchedStructlogs => { process_transactions( &block, cached_provider.get_provider().await?.deref(), @@ -77,7 +77,8 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::ServerFetchedStructlogs => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::Serverside => todo!(), }; let mut code_db = CodeDb::default(); diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e55a1ac89..4f4908eb0 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -44,9 +44,10 @@ pub enum RpcType { /// The Jumpdest source type. #[derive(ValueEnum, Clone, Debug, Copy)] pub enum JumpdestSrc { - Simulation, - Zero, - Jerigon, + ProverSimulation, + ClientFetchedStructlogs, + ServerFetchedStructlogs, // later + Serverside, // later } /// Obtain the prover input for one block diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index c8dab028c..96d30f554 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -155,7 +155,7 @@ where let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { - JumpdestSrc::Zero => { + JumpdestSrc::ClientFetchedStructlogs => { let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); futures::try_join!( tx_receipt_fut, @@ -164,12 +164,12 @@ where structlog_trace_fut, )? } - JumpdestSrc::Simulation => { + JumpdestSrc::ProverSimulation => { let (tx_receipt, pre_trace, diff_trace) = futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; (tx_receipt, pre_trace, diff_trace, None) } - JumpdestSrc::Jerigon => todo!(), + _ => todo!(), }; Ok(( From f3674090ed2e1d482c027580b96bbaf1dd7c5dd5 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 17:39:59 +0200 Subject: [PATCH 37/53] feedback --- trace_decoder/src/interface.rs | 10 ---------- zero/src/rpc/jumpdest.rs | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index e57ac7c81..1005b2afa 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -166,16 +166,6 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Review: Re-adding this. It has been removed upstream. -impl ContractCodeUsage { - /// Get code hash from a read or write operation of contract code. - pub fn get_code_hash(&self) -> H256 { - match self { - ContractCodeUsage::Read(hash) => *hash, - ContractCodeUsage::Write(bytes) => keccak(bytes), - } - } -} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index b9bf3c148..e4ef4c788 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -27,6 +27,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; +use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::trace; @@ -53,6 +54,14 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Get code hash from a read or write operation of contract code. +fn get_code_hash(usage: &ContractCodeUsage) -> H256 { + match usage { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } +} + /// Predicate that determines whether a `StructLog` that includes memory is /// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { @@ -124,7 +133,7 @@ pub(crate) fn generate_jumpdest_table( .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) .filter(|(_callee_addr, code_usage)| code_usage.is_some()) .map(|(callee_addr, code_usage)| { - (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) }) .collect(); @@ -204,6 +213,10 @@ pub(crate) fn generate_jumpdest_table( if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // Note for future debugging: There may exist edge cases, where the call + // context has been incremented before the call op fails. This should be + // accounted for before this and the following `continue`. The details are + // defined in `sys_calls.asm`. continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -217,6 +230,7 @@ pub(crate) fn generate_jumpdest_table( *address, U256::from(U160::MAX) ); + // Se note above. continue; }; let lower_20_bytes = U160::from(*address); @@ -368,6 +382,7 @@ pub(crate) fn generate_jumpdest_table( ); continue; } + assert!(jumpdest_offset.unwrap() < 24576); jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { From 464dbc0d369ab448860a49c55a64676a492e1dcd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 19:18:00 +0200 Subject: [PATCH 38/53] fix --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 25db16e19..e94b5d5a4 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -37,7 +37,7 @@ export TOKIO_WORKER_THREADS=$num_procs #export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' From 2783ccdd1dcc0124cd09c89b538c7e4731db0b90 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 23:57:10 +0200 Subject: [PATCH 39/53] debugging 460 --- .../src/generation/prover_input.rs | 24 +++++---- scripts/prove_stdio.sh | 17 +++--- scripts/test_jerigon.sh | 53 +++++++++++++++++-- scripts/test_native.sh | 4 +- trace_decoder/src/interface.rs | 2 +- zero/src/rpc/jumpdest.rs | 2 +- 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 9015ff4a3..5ae94e1b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -796,27 +796,29 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); - dbg!(&self.inputs.txn_hashes); + // dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables part2"); + info!("Generating JUMPDEST tables: Running SIM"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, simw): (Option, Option) = + let (sim, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - // assert_eq!(rw, &sw); - } + // if let (Some(rw), Some(sw)) = (rpcw, simw) { + // info!("SIMW {}", sw); + // info!("RPCW {}", rw); + // // assert_eq!(rw, &sw); + // } - info!("SIM {:#?}", sim); - info!("RPC {:#?}", rpc); + // info!("SIMW {:#?}", &simw); + // info!("RPCW {:#?}", rpcw); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index e94b5d5a4..077ce9261 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,15 +21,15 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-1}" echo "Block batch size: $BLOCK_BATCH_SIZE" -BATCH_SIZE=${BATCH_SIZE:=1} +BATCH_SIZE=${BATCH_SIZE:-1} echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" -TEST_OUT_PATH="${REPO_ROOT}/test.out" +TEST_OUT_PATH="${REPO_ROOT}/$3.test.out" # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs @@ -98,14 +98,17 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" - rm $TEST_OUT_PATH + #rm $TEST_OUT_PATH exit - else - echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." + elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then + echo -e "\n\nFailed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 + else + echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See \"zk_evm/test.out\" for more details." + exit 2 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 685939540..d1a6d272f 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -237,6 +237,51 @@ ROUND3=" 548 " +ROUND4=" +136 +186 +268 +282 +301 +304 +321 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +570 +664 +" + +ROUND5=" +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +664 +" # 470..663 from Robin for i in {470..663} @@ -249,11 +294,11 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND3" +BLOCKS="$ROUND5" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +printf "\ngithash block verdict duration\n" | tee -a witnesses/jerigon_results.txt echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do @@ -261,12 +306,12 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace + SECONDS=0 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - SECONDS=0 - timeout 10m ./prove_stdio.sh $WITNESS test_only + timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 63da02115..5e8bfeff8 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,8 +10,8 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=1 -export TOKIO_WORKER_THREADS=1 +#export RAYON_NUM_THREADS=1 +#export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full #export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 1005b2afa..35248f9a3 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -8,7 +8,7 @@ use ethereum_types::{Address, U256}; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; use evm_arithmetization::ConsolidatedHash; -use keccak_hash::{keccak, H256}; +use keccak_hash::H256; use mpt_trie::partial_trie::HashedPartialTrie; use serde::{Deserialize, Serialize}; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4ef4c788..8e1c7670f 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -33,7 +33,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; From abee81258d89d14551a674d107e9a8ca257c2f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:42:08 +0200 Subject: [PATCH 40/53] debugging 460 --- .../src/cpu/kernel/interpreter.rs | 6 ++- .../src/generation/prover_input.rs | 39 ++++++++++++------- trace_decoder/src/core.rs | 4 +- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 054c89c27..985ed2dba 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -98,7 +98,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - let jdtw = interpreter + let (a, jdtw) = interpreter .generation_state .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); @@ -106,7 +106,9 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + // (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + interpreter.generation_state.jumpdest_table = Some(a.clone()); + Some((a, jdtw)) } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 5ae94e1b2..83213783c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -359,9 +359,12 @@ impl GenerationState { } /// Returns the next used jump address. + /// todo fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // get_code from self.memory + if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -798,11 +801,14 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table; - let rpc: Option = rpcw - .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables: Running SIM"); + // let rpcw = &self.inputs.jumpdest_table.clone(); + // let rpc: Option = rpcw + // .as_ref() + // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, + // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: + // Running SIM"); + + self.inputs.jumpdest_table = None; let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); @@ -815,12 +821,15 @@ impl GenerationState { // // assert_eq!(rw, &sw); // } - // info!("SIMW {:#?}", &simw); + info!("SIMW {:#?}", &simw); // info!("RPCW {:#?}", rpcw); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &sim); + // info!("RPCP {:#?}", &rpc); + // assert_eq!(rpcw, simw); + // assert_eq!(rpc, sim); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = sim; Ok(()) } @@ -829,12 +838,12 @@ impl GenerationState { /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` pub(crate) fn set_jumpdest_analysis_inputs( - &mut self, + &self, jumpdest_table: HashMap>, - ) -> JumpDestTableWitness { + ) -> (JumpDestTableProcessed, JumpDestTableWitness) { let mut jdtw = JumpDestTableWitness::default(); - self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( - jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { + let jdtp = JumpDestTableProcessed::new(HashMap::from_iter(jumpdest_table.into_iter().map( + |(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); let code_hash = keccak(code.clone()); trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); @@ -847,9 +856,9 @@ impl GenerationState { } else { (ctx, vec![]) } - }), + }, ))); - jdtw + (jdtp, jdtw) } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 0baac3f47..ded46a83b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db.clone()); + // code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -837,7 +837,7 @@ impl Hash2Code { pub fn get(&mut self, hash: H256) -> anyhow::Result> { match self.inner.get(&hash) { Some(code) => Ok(code.clone()), - None => bail!("no code for hash {}", hash), + None => bail!("no code for hash {:#x}", hash), } } pub fn insert(&mut self, code: Vec) { From 8bfccddcac3405888318d52aae5c3654621a2509 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:44:51 +0200 Subject: [PATCH 41/53] dbg --- scripts/test_jerigon.sh | 2 +- scripts/test_native.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d1a6d272f..ed7076e30 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_JERIGON} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 5e8bfeff8..7f092f232 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_NATIVE} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 From f9c2f76f5bb2345c12e87d575bff4e27925368ee Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 14:57:32 +0200 Subject: [PATCH 42/53] bugfix --- trace_decoder/src/core.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index ded46a83b..fca50ac73 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -447,7 +447,7 @@ fn middle( storage_read, storage_written, code_usage, - self_destructed: _, + self_destructed, }, ) in traces .into_iter() @@ -544,6 +544,11 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } } + + if self_destructed { + storage_tries.remove(&keccak_hash::keccak(addr)); + state_mask.extend(state_trie.reporting_remove(addr)?) + } } jumpdest_tables.push(jumpdest_table); From 313d78d3ed9febbc5a847250b7ee47150d062650 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 15:12:21 +0200 Subject: [PATCH 43/53] dbg --- .../src/generation/prover_input.rs | 23 +++++++++---------- scripts/test_jerigon.sh | 4 ++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 83213783c..fb8f69b7d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -801,12 +801,11 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - // let rpcw = &self.inputs.jumpdest_table.clone(); - // let rpc: Option = rpcw - // .as_ref() - // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, - // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: - // Running SIM"); + let rpcw = &self.inputs.jumpdest_table.clone(); + let rpc: Option = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -822,14 +821,14 @@ impl GenerationState { // } info!("SIMW {:#?}", &simw); - // info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", rpcw); info!("SIMP {:#?}", &sim); - // info!("RPCP {:#?}", &rpc); - // assert_eq!(rpcw, simw); - // assert_eq!(rpc, sim); + info!("RPCP {:#?}", &rpc); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); - // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; - self.jumpdest_table = sim; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = sim; Ok(()) } diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index ed7076e30..781ac92ce 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -283,6 +283,10 @@ ROUND5=" 664 " +ROUND6=" +664 +" + # 470..663 from Robin for i in {470..663} do From 09079e61ba4303b0593c60fbb97763774dc57e84 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 16:53:46 +0200 Subject: [PATCH 44/53] fix --- trace_decoder/src/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index fca50ac73..4b3303ddd 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - // code.extend(code_db.clone()); + code.extend(code_db.clone()); let OtherBlockData { b_data: From 7c84a63a6d4318fa360c7c04d4b024c4d7bf83b3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 17:58:11 +0200 Subject: [PATCH 45/53] batching working --- .../src/generation/prover_input.rs | 17 +++++++++-------- scripts/test_jerigon.sh | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index fb8f69b7d..425df5397 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -802,18 +802,19 @@ impl GenerationState { // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table.clone(); - let rpc: Option = rpcw + let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, ref simw): (Option, Option) = + let (simp, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + info!("Generating JUMPDEST tables: finished"); + // if let (Some(rw), Some(sw)) = (rpcw, simw) { // info!("SIMW {}", sw); // info!("RPCW {}", rw); @@ -822,12 +823,12 @@ impl GenerationState { info!("SIMW {:#?}", &simw); info!("RPCW {:#?}", rpcw); - info!("SIMP {:#?}", &sim); - info!("RPCP {:#?}", &rpc); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &simp); + info!("RPCP {:#?}", &rpcp); + // assert_eq!(simw, rpcw); + // assert_eq!(simp, rpcp); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; // self.jumpdest_table = sim; Ok(()) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 781ac92ce..8e02977c8 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -298,7 +298,7 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND5" +BLOCKS="$ROUND6" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" From 4202ece130bcb221adb1152aed29d9fc599d65f4 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 22:04:26 +0200 Subject: [PATCH 46/53] cleanups --- .../src/generation/prover_input.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 425df5397..f3b5ec54a 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -799,9 +799,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table.clone(); + let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -815,21 +813,12 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - // if let (Some(rw), Some(sw)) = (rpcw, simw) { - // info!("SIMW {}", sw); - // info!("RPCW {}", rw); - // // assert_eq!(rw, &sw); - // } - info!("SIMW {:#?}", &simw); - info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", &rpcw); info!("SIMP {:#?}", &simp); info!("RPCP {:#?}", &rpcp); - // assert_eq!(simw, rpcw); - // assert_eq!(simp, rpcp); self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - // self.jumpdest_table = sim; Ok(()) } From e1124d3c631aae1123370c3a7c2878b079bfbc5c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 14:32:13 +0200 Subject: [PATCH 47/53] feedback docs --- .../src/generation/jumpdest.rs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 9a3201cce..1966c265d 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -1,3 +1,27 @@ +//! EVM opcode 0x5B or 91 is [`JUMPDEST`] which encodes a a valid offset, that +//! opcodes `JUMP` and `JUMPI` can jump to. Jumps to non-[`JUMPDEST`] +//! instructions are invalid. During an execution a [`JUMPDEST`] may be visited +//! zero or more times. Offsets are measured in bytes with respect to the +//! beginning of some contract code, which is uniquely identified by its +//! `CodeHash`. Every time control flow is switches to another contract through +//! a `CALL`-like instruction a new call context, `Context`, is created. Thus, +//! the tripple (`CodeHash`,`Context`, `Offset`) uniquely identifies an visited +//! [`JUMPDEST`] offset of an execution. +//! +//! Since an operation like e.g. `PUSH 0x5B` does not encode a valid +//! [`JUMPDEST`] in its second byte, and `PUSH32 +//! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not +//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! when proving validity of jump operations. +//! +//! This module concerns itself with data structures for collecting these +//! offsets for [`JUMPDEST`] that was visited during an execution and are not +//! recording duplicity. The proofs that each of these offsets are not rendered +//! invalid by any of the previous 32 bytes `PUSH1`-`PUSH32` is computed later +//! in [`prove_context_jumpdests`] on basis of these collections. +//! +//! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b + use std::cmp::max; use std::{ collections::{BTreeSet, HashMap}, @@ -9,12 +33,13 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, each time creating a new -/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +/// Each `CodeAddress` can be called one or more times, +/// each time creating a new `Context`. +/// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct ContextJumpDests(pub HashMap>); -/// The result after proving a `JumpDestTableWitness`. +/// The result after proving a [`JumpDestTableWitness`]. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); From 1f8d476c869ff0567d5bb1d6ce52e7c8fb63be48 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 15:20:27 +0200 Subject: [PATCH 48/53] feedback --- evm_arithmetization/src/generation/jumpdest.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 1966c265d..a7a104a15 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -11,7 +11,7 @@ //! Since an operation like e.g. `PUSH 0x5B` does not encode a valid //! [`JUMPDEST`] in its second byte, and `PUSH32 //! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not -//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! encode valid [`JUMPDESTS`] in bytes 1-32, some diligence must be exercised //! when proving validity of jump operations. //! //! This module concerns itself with data structures for collecting these @@ -64,10 +64,6 @@ impl JumpDestTableProcessed { } impl JumpDestTableWitness { - pub fn new(ctx_map: HashMap) -> Self { - Self(ctx_map) - } - pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { self.0.get(code_hash) } From 7319a15a3eb99493822d7a72fae09d1d6e1acce1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:06:59 +0200 Subject: [PATCH 49/53] feedback filtermap --- evm_arithmetization/src/generation/jumpdest.rs | 4 ++-- zero/src/rpc/jumpdest.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index a7a104a15..f980d51ac 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -33,7 +33,7 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, +/// Each `CodeHash` can be called one or more times, /// each time creating a new `Context`. /// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] @@ -43,7 +43,7 @@ pub struct ContextJumpDests(pub HashMap>); #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); -/// Map `CodeAddress -> (Context -> [JumpDests])` +/// Map `CodeHash -> (Context -> [JumpDests])` #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct JumpDestTableWitness(HashMap); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8e1c7670f..e4b8e5e29 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -127,16 +127,19 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); let mut code_db = CodeDb::default(); - // This map does not contain `initcodes`. + // This map does neither contain the `init` field of Contract Deployment + // transactions nor CREATE, CREATE2 payloads. let callee_addr_to_code_hash: HashMap = tx_traces .iter() - .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) - .filter(|(_callee_addr, code_usage)| code_usage.is_some()) - .map(|(callee_addr, code_usage)| { - (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) + .filter_map(|(callee_addr, trace)| { + trace + .code_usage + .as_ref() + .map(|code| (*callee_addr, get_code_hash(&code))) }) .collect(); + // REVIEW: will be removed before merge trace!( "Transaction: {} is a {}.", tx.hash, @@ -166,9 +169,6 @@ pub(crate) fn generate_jumpdest_table( // true condition and target `jump_target`. let mut prev_jump: Option = None; - // Call depth of the previous `entry`. We initialize to 0 as this compares - // smaller to 1. - //let mut prev_depth = 0; // The next available context. Starts at 1. Never decrements. let mut next_ctx_available = 1; // Immediately use context 1; From d4838e0c83191afbe34cfd050b1efd1f3fdc49f3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:43:28 +0200 Subject: [PATCH 50/53] review --- zero/src/rpc/jumpdest.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4b8e5e29..3997e545c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -191,6 +191,7 @@ pub(crate) fn generate_jumpdest_table( ); let (code_hash, ctx) = call_stack.last().unwrap(); + // REVIEW: will be removed before merge trace!("TX: {:?}", tx.hash); trace!("STEP: {:?}", step); trace!("STEPS: {:?}", struct_log.len()); From 27b5719457015015df0a1bf0be7cc51cde2d681f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 12:26:57 +0200 Subject: [PATCH 51/53] fmt --- zero/src/rpc/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3997e545c..14b133705 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -135,7 +135,7 @@ pub(crate) fn generate_jumpdest_table( trace .code_usage .as_ref() - .map(|code| (*callee_addr, get_code_hash(&code))) + .map(|code| (*callee_addr, get_code_hash(code))) }) .collect(); From eaf3ed7175f3de8c196b709691b3eabc38654e89 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 13:35:14 +0200 Subject: [PATCH 52/53] fix set_jumpdest_analysis_inputs_rpc --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 4 ++-- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 3 ++- evm_arithmetization/src/generation/prover_input.rs | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 985ed2dba..9ff445fa2 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -100,7 +100,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let (a, jdtw) = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); + .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", @@ -163,7 +163,7 @@ pub(crate) fn set_registers_and_run( /// /// - `jumpdest_table_rpc`: The raw table received from RPC. /// - `code_db`: The corresponding database of contract code used in the trace. -pub(crate) fn set_jumpdest_analysis_inputs_rpc( +pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, ) -> JumpDestTableProcessed { diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index 607f8cc69..d1b076118 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -17,7 +17,8 @@ use crate::witness::operation::CONTEXT_SCALING_FACTOR; impl Interpreter { pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); + let (jdtp, _jdtw) = self.generation_state.get_jumpdest_analysis_inputs(jumps); + self.generation_state.jumpdest_table = Some(jdtp); } pub(crate) fn get_jumpdest_bit(&self, offset: usize) -> U256 { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index f3b5ec54a..e4206cb15 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -26,7 +26,7 @@ use crate::cpu::kernel::constants::cancun_constants::{ }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::interpreter::{ - set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, + get_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, }; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; @@ -798,11 +798,13 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call + // REVIEW: This will be rewritten to only run simulation when + // `self.inputs.jumpdest_table` is `None`. info!("Generating JUMPDEST tables"); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -826,7 +828,7 @@ impl GenerationState { /// Given a HashMap containing the contexts and the jumpdest addresses, /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` - pub(crate) fn set_jumpdest_analysis_inputs( + pub(crate) fn get_jumpdest_analysis_inputs( &self, jumpdest_table: HashMap>, ) -> (JumpDestTableProcessed, JumpDestTableWitness) { From 10b6a22cd4b9826ad7f8c77e02bc14872c15f112 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:03:41 +0200 Subject: [PATCH 53/53] discuss: deser in #427 (#681) * refactor: StructPogPrime * keep module doc, avoid name `vec` --- zero/src/rpc/jumpdest.rs | 192 ++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 116 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 14b133705..3f5057b27 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -13,6 +13,7 @@ use alloy::primitives::U160; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::GethTrace; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; use alloy::transports::RpcError; @@ -24,7 +25,6 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::CodeDb; use keccak_hash::keccak; -use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; @@ -86,7 +86,7 @@ where .await?; let stackonly_structlog_opt: Option> = - normalize_structlog(&stackonly_structlog_trace).await; + trace2structlog(stackonly_structlog_trace).unwrap_or_default(); let need_memory = stackonly_structlog_opt .as_deref() @@ -102,7 +102,7 @@ where structlog_tracing_options(true, need_memory, false), ); - let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + let memory_structlog = trace2structlog(memory_structlog_fut.await?).unwrap_or_default(); Ok::>, RpcError>(memory_structlog) }; @@ -398,131 +398,91 @@ pub(crate) fn generate_jumpdest_table( Ok((jumpdest_table, code_db)) } +fn trace2structlog(trace: GethTrace) -> Result>, serde_json::Error> { + match trace { + GethTrace::Default(it) => Ok(Some(it.struct_logs)), + GethTrace::JS(it) => Ok(Some(compat::deserialize(it)?.struct_logs)), + _ => Ok(None), + } +} /// This module exists as a workaround for parsing `StructLog`. The `error` -/// field is a string in Alloy but an object in Erigon. -pub mod structlogprime { - use core::option::Option::None; - use std::collections::BTreeMap; +/// field is a string in Geth and Alloy but an object in Erigon. A PR[^1] has +/// been merged to fix this upstream and should eventually render this +/// unnecessary. [^1]: `https://github.com/erigontech/erigon/pull/12089` +mod compat { + use std::{collections::BTreeMap, fmt, iter}; - use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; + use alloy::rpc::types::trace::geth::{DefaultFrame, StructLog}; use alloy_primitives::{Bytes, B256, U256}; - use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; - use serde_json::Value; - - /// Geth Default struct log trace frame - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub(crate) struct DefaultFramePrime { - /// Whether the transaction failed - pub failed: bool, - /// How much gas was used. - pub gas: u64, - /// Output of the transaction - #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] - pub return_value: Bytes, - /// Recorded traces of the transaction - pub struct_logs: Vec, - } + use serde::{de::SeqAccess, Deserialize, Deserializer}; - /// Represents a struct log entry in a trace - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - pub(crate) struct StructLogPrime { - /// Program counter - pub pc: u64, - /// Opcode to be executed - pub op: String, - /// Remaining gas - pub gas: u64, - /// Cost for executing op - #[serde(rename = "gasCost")] - pub gas_cost: u64, - /// Current call depth - pub depth: u64, - /// Error message if any - #[serde(default, skip)] - pub error: Option, - /// EVM stack - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stack: Option>, - /// Last call's return data. Enabled via enableReturnData - #[serde( - default, - rename = "returnData", - skip_serializing_if = "Option::is_none" - )] - pub return_data: Option, - /// ref - #[serde(default, skip_serializing_if = "Option::is_none")] - pub memory: Option>, - /// Size of memory. - #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] - pub memory_size: Option, - /// Storage slots of current contract read from and written to. Only - /// emitted for SLOAD and SSTORE. Disabled via disableStorage - #[serde( - default, - skip_serializing_if = "Option::is_none", - serialize_with = "serialize_string_storage_map_opt" - )] - pub storage: Option>, - /// Refund counter - #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] - pub refund_counter: Option, + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + _DefaultFrame::deserialize(d) } - /// Serializes a storage map as a list of key-value pairs _without_ - /// 0x-prefix - pub(crate) fn serialize_string_storage_map_opt( - storage: &Option>, - s: S, - ) -> Result { - match storage { - None => s.serialize_none(), - Some(storage) => { - let mut m = s.serialize_map(Some(storage.len()))?; - for (key, val) in storage.iter() { - let key = format!("{:?}", key); - let val = format!("{:?}", val); - // skip the 0x prefix - m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; - } - m.end() - } + /// The `error` field is a `string` in `geth` etc. but an `object` in + /// `erigon`. + fn error<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Error { + String(String), + #[allow(dead_code)] + Object(serde_json::Map), } + Ok(match Error::deserialize(d)? { + Error::String(it) => Some(it), + Error::Object(_) => None, + }) } - impl TryInto for DefaultFramePrime { - fn try_into(self) -> Result { - let a = serde_json::to_string(&self)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let c = serde_json::to_string(&b)?; - let d: DefaultFrame = serde_json::from_str(&c)?; - Ok(d) - } - - type Error = anyhow::Error; + #[derive(Deserialize)] + #[serde(remote = "DefaultFrame", rename_all = "camelCase")] + struct _DefaultFrame { + failed: bool, + gas: u64, + return_value: Bytes, + #[serde(deserialize_with = "vec_structlog")] + struct_logs: Vec, } - pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { - let a = serde_json::to_string(structlog_object)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let d: DefaultFrame = b.try_into()?; - Ok(d) + fn vec_structlog<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an array of `StructLog`") + } + fn visit_seq>(self, mut seq: A) -> Result { + #[derive(Deserialize)] + struct With(#[serde(with = "_StructLog")] StructLog); + let v = iter::from_fn(|| seq.next_element().transpose()) + .map(|it| it.map(|With(it)| it)) + .collect::>()?; + Ok(v) + } + } + + d.deserialize_seq(Visitor) } - pub(crate) async fn normalize_structlog( - unnormalized_structlog: &GethTrace, - ) -> Option> { - match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - } + #[derive(Deserialize)] + #[serde(remote = "StructLog", rename_all = "camelCase")] + struct _StructLog { + pc: u64, + op: String, + gas: u64, + gas_cost: u64, + depth: u64, + #[serde(deserialize_with = "error")] + error: Option, + stack: Option>, + return_data: Option, + memory: Option>, + #[serde(rename = "memSize")] + memory_size: Option, + storage: Option>, + #[serde(rename = "refund")] + refund_counter: Option, } }