Merge pull request #109 from EspressoSystems/update-vid

Update VID to use Bn254, fix E2E test

.github/workflows/espresso-e2e.yml

@@ -128,9 +128,9 @@ jobs:
       - name: Install docker-compose
         uses: KengoTODA/[email protected]
         with:
-          version: '2.14.2'
+          version: '2.22.0'
 
       - name: Run test
         run: |
           packages=`go list ./... | grep system_tests`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -timeout 25m ./... -run TestEspressoE2E
+          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -v -timeout 25m ./... -run TestEspressoE2E

arbitrator/vid-helper/Cargo.toml

@@ -5,18 +5,21 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-ark-bls12-381 = "0.4.0"
+ark-bn254 = "0.4"
 ark-serialize = { version = "0.4", features = ["derive"] }
 base64 = "0.22"
 derive_more = "0.99.17"
+base64-bytes = "0.1"
 serde_json = "1.0"
 tagged-base64 = { git = "https://github.com/EspressoSystems/tagged-base64", tag = "0.3.4" }
 lazy_static = "1.4"
 serde = { version = "1.0", features = ["derive"] }
 sha2 = { version = "0.10.1" }
-jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", features = [
+jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.2", features = [
     "test-srs", "std"
 ], default-features = false}
+jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.2", features = [
+], default-features = false}
 trait-set = "0.3.0"
 num-traits = "0.2.17"
 derivative = "2.2"

arbitrator/vid-helper/src/lib.rs

@@ -1,7 +1,7 @@
 mod bytes;
 mod namespace;
 
-use ark_bls12_381::Bls12_381;
+use ark_bn254::Bn254;
 use ark_serialize::CanonicalDeserialize;
 use jf_primitives::{
     pcs::prelude::UnivariateUniversalParams,
@@ -14,7 +14,7 @@ use tagged_base64::TaggedBase64;
 
 use crate::bytes::Bytes;
 
-pub type VidScheme = Advz<Bls12_381, sha2::Sha256>;
+pub type VidScheme = Advz<Bn254, sha2::Sha256>;
 
 lazy_static! {
     // Initialize the byte array from JSON content
@@ -49,10 +49,9 @@ pub fn verify_namespace_helper(
     let tagged = TaggedBase64::parse(&commit_str).unwrap();
     let commit: <VidScheme as VidSchemeTrait>::Commit = tagged.try_into().unwrap();
 
-    let srs = UnivariateUniversalParams::<Bls12_381>::deserialize_uncompressed_unchecked(
-        SRS_VEC.as_slice(),
-    )
-    .unwrap();
+    let srs =
+        UnivariateUniversalParams::<Bn254>::deserialize_uncompressed_unchecked(SRS_VEC.as_slice())
+            .unwrap();
     let num_storage_nodes = match &proof {
         NamespaceProof::Existence { vid_common, .. } => {
             VidScheme::get_num_storage_nodes(&vid_common)
@@ -81,12 +80,31 @@ fn hash_txns(namespace: u64, txns: &[Transaction]) -> String {
     format!("{:x}", hash_result)
 }
 
-#[test]
-fn test_verify_namespace_helper() {
-    let proof_bytes = b"{\"NonExistence\":{\"ns_id\":0}}";
-    let commit_bytes = b"HASH~1yS-KEtL3oDZDBJdsW51Pd7zywIiHesBZsTbpOzrxOfu";
-    let txn_comm_str = hash_txns(0, &[]);
-    let txn_comm_bytes = txn_comm_str.as_bytes();
-    let ns_table_bytes = &[0, 0, 0, 0];
-    verify_namespace_helper(0, proof_bytes, commit_bytes, ns_table_bytes, txn_comm_bytes);
+#[cfg(test)]
+mod test {
+    use super::*;
+    use ark_serialize::CanonicalSerialize;
+    use jf_primitives::pcs::{
+        checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme,
+    };
+    #[test]
+    fn test_verify_namespace_helper() {
+        let proof_bytes = b"{\"NonExistence\":{\"ns_id\":0}}";
+        let commit_bytes = b"HASH~1yS-KEtL3oDZDBJdsW51Pd7zywIiHesBZsTbpOzrxOfu";
+        let txn_comm_str = hash_txns(0, &[]);
+        let txn_comm_bytes = txn_comm_str.as_bytes();
+        let ns_table_bytes = &[0, 0, 0, 0];
+        verify_namespace_helper(0, proof_bytes, commit_bytes, ns_table_bytes, txn_comm_bytes);
+    }
+
+    #[test]
+    fn write_srs_to_file() {
+        let mut bytes = Vec::new();
+        let mut rng = jf_utils::test_rng();
+        UnivariateKzgPCS::<Bn254>::gen_srs_for_testing(&mut rng, checked_fft_size(200).unwrap())
+            .unwrap()
+            .serialize_uncompressed(&mut bytes)
+            .unwrap();
+        let _ = std::fs::write("srs.json", serde_json::to_string(&bytes).unwrap());
+    }
 }

arbitrator/vid-helper/src/namespace.rs

@@ -1,7 +1,7 @@
 // TODO import from sequencer: https://github.com/EspressoSystems/nitro-espresso-integration/issues/87
 // This module is essentially copy and pasted VID logic from the sequencer repo. It is an unfortunate workaround
 // until the VID portion of the sequencer repo is WASM-compatible.
-use ark_bls12_381::Bls12_381;
+use ark_bn254::Bn254;
 use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use core::fmt;
 use derivative::Derivative;
@@ -198,7 +198,7 @@ impl TryFrom<TxTableEntry> for NamespaceId {
     }
 }
 
-pub type VidScheme = jf_primitives::vid::advz::Advz<ark_bls12_381::Bls12_381, sha2::Sha256>;
+pub type VidScheme = jf_primitives::vid::advz::Advz<ark_bn254::Bn254, sha2::Sha256>;
 
 /// Namespace proof type
 ///
@@ -215,13 +215,14 @@ pub type VidScheme = jf_primitives::vid::advz::Advz<ark_bls12_381::Bls12_381, sh
 /// ```
 /// but that's still pretty crufty.
 pub type JellyfishNamespaceProof =
-    LargeRangeProof<<UnivariateKzgPCS<Bls12_381> as PolynomialCommitmentScheme>::Evaluation>;
+    LargeRangeProof<<UnivariateKzgPCS<Bn254> as PolynomialCommitmentScheme>::Evaluation>;
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(bound = "")] // for V
 pub enum NamespaceProof {
     Existence {
-        ns_payload_flat: Bytes,
+        #[serde(with = "base64_bytes")]
+        ns_payload_flat: Vec<u8>,
         ns_id: NamespaceId,
         ns_proof: JellyfishNamespaceProof,
         vid_common: <VidScheme as VidSchemeTrait>::Common,

cmd/replay/main.go

@@ -302,6 +302,13 @@ func main() {
 			if !commitment.Equals(hotshotHeader.Commit()) {
 				panic(fmt.Sprintf("invalid hotshot header jst header at %v expected: %v, provided %v.", height, hotshotHeader.Commit(), commitment))
 			}
+			if jst.BlockMerkleProof == nil {
+				panic("block merkle proof missing from justification")
+			}
+			_, err = jst.BlockMerkleProof.Verify(commitment)
+			if err != nil {
+				panic("merkle proof verification failure")
+			}
 			arbvid.VerifyNamespace(chainConfig.ChainID.Uint64(), *jst.Proof, *jst.Header.PayloadCommitment, *jst.Header.NsTable, txs)
 		}
 

system_tests/espresso-e2e/.env

@@ -6,6 +6,11 @@ RUST_LOG_FORMAT=full
 
 L1_BLOCK_TIME_SEC=3
 
+# Parallelism config
+# Since we have many processes using async-std, limit them to one thread each to keep the CPU
+# requirements for the local demo down.
+ASYNC_STD_THREAD_COUNT=1
+
 # Internal port inside container
 ESPRESSO_WEB_SERVER_PORT=40000
 ESPRESSO_ORCHESTRATOR_PORT=40001
@@ -25,9 +30,15 @@ ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://host.docker.internal:$ESPRESSO_SEQUENCER_
 ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG=true
 ESPRESSO_COMMITMENT_ETH_MNEMONIC="indoor dish desk flag debris potato excuse depart ticket judge file exit"
 ESPRESSO_SEQUENCER_ETH_MNEMONIC="test test test test test test test test test test test junk"
-ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5
+ESPRESSO_DEPLOYER_ACCOUNT_INDEX=5
 ESPRESSO_COMMITMENT_TASK_PORT=60000
 
+# Ethereum accounts (note 11-15 are used by the sequencer nodes)
+ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5
+
+# Conctracts
+ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS=0x217788c286797d56cd59af5e493f3699c39cbbe8
+
 # Example sequencer demo private keys
 ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0=BLS_SIGNING_KEY~lNDh4Pn-pTAyzyprOAFdXHwhrKhEwqwtMtkD3CZF4x3o
 ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_1=BLS_SIGNING_KEY~-DO72m_SFl6NQMYknm05FYpPEklkeqz-B3g2mFdbuS83

system_tests/espresso-e2e/docker-compose.yaml

@@ -1,5 +1,21 @@
 version: '3.9'
 services:
+  deploy-contracts:
+    image: ghcr.io/espressosystems/espresso-sequencer/deploy:arbitrum-integrationmusl
+    environment:
+      - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL
+      - ESPRESSO_SEQUENCER_L1_PROVIDER
+      - ESPRESSO_SEQUENCER_ETH_MNEMONIC=$ESPRESSO_COMMITMENT_ETH_MNEMONIC
+      - ESPRESSO_DEPLOYER_ACCOUNT_INDEX
+      - RUST_LOG
+      - RUST_LOG_FORMAT
+      - ASYNC_STD_THREAD_COUNT
+    depends_on:
+      orchestrator:
+        condition: service_healthy
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   orchestrator:
     image: ghcr.io/espressosystems/espresso-sequencer/orchestrator:arbitrum-integrationmusl
     ports:
@@ -113,14 +129,13 @@ services:
     image: ghcr.io/espressosystems/espresso-sequencer/commitment-task:arbitrum-integrationmusl
     ports:
       - "$ESPRESSO_COMMITMENT_TASK_PORT:$ESPRESSO_COMMITMENT_TASK_PORT"
-    command: commitment-task --deploy
     environment:
       - ESPRESSO_SEQUENCER_ETH_MNEMONIC=$ESPRESSO_COMMITMENT_ETH_MNEMONIC
       - ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX
       - ESPRESSO_COMMITMENT_TASK_PORT
       - ESPRESSO_SEQUENCER_URL
       - ESPRESSO_SEQUENCER_L1_PROVIDER
-      - ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG
+      - ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS
       - RUST_LOG
       - RUST_LOG_FORMAT
     depends_on:
@@ -130,6 +145,9 @@ services:
         condition: service_healthy
       da-server:
         condition: service_healthy
+      deploy-contracts:
+        condition: service_completed_successfully
+
     extra_hosts:
       - "host.docker.internal:host-gateway"