Develop (#57)

* Optimize get ver (#48) * optimize get_ver() * Create rust.yml * fixed clippy and removed dead code Co-authored-by: harry <[email protected]> Co-authored-by: simonjiao <[email protected]> Co-authored-by: simonjiao <[email protected]> * add clean_aux() (#49) Co-authored-by: harry <[email protected]> * adding original errors to new mapped error messages (#51) * Prepare a version of `storage` with Ruc 3.0.0 (#52) * ruc to 3.0.0 * Update rust.yml Co-authored-by: onewayfunc <[email protected]> * fmerk update to v2.1.1 * rollback public interface and ruc version * fix clippy * update clean_aux * V1.0.0 archive (#63) (#66) * catch up fix * add back deprecated API Co-authored-by: Charlie Chen <[email protected]> * Fo 996 v2 (#60) * support `state_at` for Storage * fix test errors * update workflow * rollback public interface * fix iterate * fix some test cases * add snapshot metadata * build snapshots in loading phase * build snapshots in commit phase * enable snapshot in get_ver * add more test cases * add option to construct_base * FO-1238: add merk db all iter and keys internal cf -> keys aux cf base prefix * enable construct base * fix clean_aux and more test cases * cosntrcut_base in clean_aux * avoid panic and add more logs Co-authored-by: Charlie Chen <[email protected]> Co-authored-by: simonjiao <[email protected]> Co-authored-by: 18328594608 <[email protected]> Co-authored-by: Charlie Chen <[email protected]> Co-authored-by: harry <[email protected]> Co-authored-by: simonjiao <[email protected]> Co-authored-by: Harshad Patil <[email protected]> Co-authored-by: Weikeng Chen <[email protected]> Co-authored-by: onewayfunc <[email protected]> Co-authored-by: HarryLiIsMe <[email protected]> Co-authored-by: Charlie Chen <[email protected]> Co-authored-by: 18328594608 <[email protected]>
FindoraNetwork · Nov 25, 2022 · 8188cff · 8188cff
1 parent d737551
commit 8188cff
Show file tree

Hide file tree

Showing 15 changed files with 2,156 additions and 107 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -2,7 +2,7 @@ name: Rust
 
 on:
   push:
-    branches: [ "master" ]
+    branches: [ "master", "develop" ]
   pull_request:
     branches: [ "master" ]
 

diff --git a/fin_db/Cargo.toml b/fin_db/Cargo.toml
@@ -6,8 +6,8 @@ edition = "2021"
 
 [dependencies]
 ruc = "1.0"
-fmerk = "0.1"
+fmerk = { git = "https://github.com/FindoraNetwork/fmerk.git", tag = "v2.1.1"}
 storage = { path = "../storage", version = "0.2" }
 
 [features]
-iterator = ["storage/iterator"]
+iterator = ["storage/iterator"]
diff --git a/fin_db/src/lib.rs b/fin_db/src/lib.rs
@@ -1,4 +1,8 @@
-use fmerk::{rocksdb, tree::Tree, BatchEntry, Merk, Op};
+use fmerk::{
+    rocksdb::{self},
+    tree::Tree,
+    BatchEntry, Merk, Op,
+};
 use ruc::*;
 use std::path::{Path, PathBuf};
 use storage::db::{DbIter, IterOrder, KVBatch, KValue, MerkleDB};
@@ -28,13 +32,15 @@ impl FinDB {
     ///
     /// path, one will be created.
     pub fn open<P: AsRef<Path>>(path: P) -> Result<FinDB> {
-        let db = Merk::open(path).map_err(|_| eg!("Failed to open db"))?;
+        let db = Merk::open(path).map_err(|e| eg!("Failed to open db {}", e))?;
         Ok(Self { db })
     }
 
     /// Closes db and deletes all data from disk.
     pub fn destroy(self) -> Result<()> {
-        self.db.destroy().map_err(|_| eg!("Failed to destory db"))
+        self.db
+            .destroy()
+            .map_err(|e| eg!("Failed to destory db {}", e))
     }
 }
 
@@ -52,14 +58,14 @@ impl MerkleDB for FinDB {
     fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
         self.db
             .get(key)
-            .map_err(|_| eg!("Failed to get data from db"))
+            .map_err(|e| eg!("Failed to get data from db {}", e))
     }
 
     /// Gets an auxiliary value.
     fn get_aux(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
         self.db
             .get_aux(key)
-            .map_err(|_| eg!("Failed to get aux from db"))
+            .map_err(|e| eg!("Failed to get aux from db {}", e))
     }
 
     /// Puts a batch of KVs
@@ -93,17 +99,26 @@ impl MerkleDB for FinDB {
             IterOrder::Desc => Box::new(self.db.iter_opt_aux(rocksdb::IteratorMode::End, readopts)),
         }
     }
+    fn db_all_iterator(&self, order: IterOrder) -> DbIter<'_>
+    {
+        let readopts = rocksdb::ReadOptions::default();
+        match order {
+            IterOrder::Asc => Box::new(self.db.iter_opt(rocksdb::IteratorMode::Start, readopts)),
+            IterOrder::Desc => Box::new(self.db.iter_opt(rocksdb::IteratorMode::End, readopts)),
+        }
+    }
+
 
     /// Commits changes.
     fn commit(&mut self, aux: KVBatch, flush: bool) -> Result<()> {
         let batch_aux = to_batch(aux);
         self.db
             .commit(batch_aux.as_ref())
-            .map_err(|_| eg!("Failed to commit to db"))?;
+            .map_err(|e| eg!("Failed to commit to db {}", e))?;
         if flush {
             self.db
                 .flush()
-                .map_err(|_| eg!("Failed to flush memtables"))?;
+                .map_err(|e| eg!("Failed to flush memtables {}", e))?;
         }
         Ok(())
     }
@@ -112,7 +127,7 @@ impl MerkleDB for FinDB {
     fn snapshot<P: AsRef<Path>>(&self, path: P) -> Result<()> {
         self.db
             .snapshot(path)
-            .map_err(|_| eg!("Failed to take snapshot"))?;
+            .map_err(|e| eg!("Failed to take snapshot {}", e))?;
         Ok(())
     }
 
@@ -121,6 +136,10 @@ impl MerkleDB for FinDB {
         let kv = Tree::decode(kv_pair.0.to_vec(), &kv_pair.1);
         (kv.key().to_vec(), kv.value().to_vec())
     }
+
+    fn clean_aux(&mut self) -> Result<()> {
+        self.db.clean_aux().map_err(|e| eg!(e))
+    }
 }
 
 /// Rocks db
@@ -246,6 +265,15 @@ impl MerkleDB for RocksDB {
         self.iter(lower, upper, order)
     }
 
+    fn db_all_iterator(&self, order: IterOrder) -> DbIter<'_>
+    {
+        let readopts = rocksdb::ReadOptions::default();
+        match order {
+            IterOrder::Asc => Box::new(self.iter_opt(rocksdb::IteratorMode::Start, readopts)),
+            IterOrder::Desc => Box::new(self.iter_opt(rocksdb::IteratorMode::End, readopts)),
+        }
+    }
+
     /// Commits changes.
     fn commit(&mut self, kvs: KVBatch, flush: bool) -> Result<()> {
         // write batch
@@ -255,7 +283,7 @@ impl MerkleDB for RocksDB {
         if flush {
             self.db
                 .flush()
-                .map_err(|_| eg!("Failed to flush memtables"))?;
+                .map_err(|e| eg!("Failed to flush memtables {}", e))?;
         }
 
         Ok(())
@@ -273,4 +301,19 @@ impl MerkleDB for RocksDB {
     fn decode_kv(&self, kv_pair: (Box<[u8]>, Box<[u8]>)) -> KValue {
         (kv_pair.0.to_vec(), kv_pair.1.to_vec())
     }
+
+    fn clean_aux(&mut self) -> Result<()> {
+        // let state_cf = self.db.cf_handle(CF_STATE).unwrap();
+        // let mut batch = rocksdb::WriteBatch::default();
+        // for (key, _) in self.db.iterator_cf(state_cf, IteratorMode::Start) {
+        //     batch.delete_cf(state_cf, key);
+        // }
+
+        // let mut opts = rocksdb::WriteOptions::default();
+        // opts.set_sync(false);
+        // self.db.write_opt(batch, &opts).c(d!())?;
+        // self.db.flush_cf(state_cf).c(d!())?;
+
+        Ok(())
+    }
 }
diff --git a/mem_db/src/lib.rs b/mem_db/src/lib.rs
@@ -84,6 +84,28 @@ impl MerkleDB for MemoryDB {
         Ok(())
     }
 
+    fn db_all_iterator(&self, order: IterOrder) -> DbIter<'_>
+    {
+        let lower_key: &[u8] = b"0";
+
+        let lower = lower_key.to_vec().into_boxed_slice();
+        let upper =  lower_key.to_vec().into_boxed_slice();
+
+        match order {
+            IterOrder::Asc => Box::new(
+                self.inner
+                    .range::<Box<[u8]>, _>((Included(&lower), Excluded(&upper)))
+                    .filter_map(|(k, v)| v.as_ref().map(|v| (k.clone(), v.clone()))),
+            ),
+            IterOrder::Desc => Box::new(
+                self.inner
+                    .range::<Box<[u8]>, _>((Included(&lower), Excluded(&upper)))
+                    .filter_map(|(k, v)| v.as_ref().map(|v| (k.clone(), v.clone())))
+                    .rev(),
+            ),
+        }
+    }
+
     fn iter(&self, lower: &[u8], upper: &[u8], order: IterOrder) -> DbIter<'_> {
         let lower = lower.to_vec().into_boxed_slice();
         let upper = upper.to_vec().into_boxed_slice();
@@ -142,6 +164,11 @@ impl MerkleDB for MemoryDB {
     fn decode_kv(&self, kv_pair: (Box<[u8]>, Box<[u8]>)) -> KValue {
         (kv_pair.0.to_vec(), kv_pair.1.to_vec())
     }
+
+    fn clean_aux(&mut self) -> Result<()> {
+        self.aux.clear();
+        Ok(())
+    }
 }
 
 impl Drop for MemoryDB {

diff --git a/storage/Cargo.toml b/storage/Cargo.toml
@@ -17,4 +17,6 @@ temp_db = { path = "../temp_db", version = "0.2" }
 mem_db = { path = "../mem_db", version = "0.2" }
 
 [features]
+default = [ "optimize_get_ver" ]
 iterator = []
+optimize_get_ver = []
diff --git a/storage/src/db.rs b/storage/src/db.rs
@@ -9,6 +9,7 @@ pub type KVEntry = (StoreKey, Option<Vec<u8>>);
 pub type KVBatch = Vec<KVEntry>;
 pub type DbIter<'a> = Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a>;
 
+#[derive(Debug)]
 pub enum IterOrder {
     Asc,
     Desc,
@@ -28,13 +29,18 @@ pub trait MerkleDB {
 
     fn iter_aux(&self, lower: &[u8], upper: &[u8], order: IterOrder) -> DbIter<'_>;
 
+    fn db_all_iterator(&self, order: IterOrder) -> DbIter<'_>;
+
     fn commit(&mut self, kvs: KVBatch, flush: bool) -> Result<()>;
 
     fn snapshot<P: AsRef<Path>>(&self, path: P) -> Result<()>;
 
     fn decode_kv(&self, kv_pair: (Box<[u8]>, Box<[u8]>)) -> KValue;
 
+    #[inline]
     fn as_mut(&mut self) -> &mut Self {
         self
     }
+
+    fn clean_aux(&mut self) -> Result<()>;
 }
diff --git a/storage/src/lib.rs b/storage/src/lib.rs
@@ -1,4 +1,111 @@
 /// The merkle db
+///
+#[deny(
+////The following are allowed by default lints according to
+////https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html
+absolute_paths_not_starting_with_crate,
+//box_pointers, async trait must use it
+//elided_lifetimes_in_paths, //allow anonymous lifetime
+explicit_outlives_requirements,
+keyword_idents,
+macro_use_extern_crate,
+meta_variable_misuse,
+missing_abi,
+//missing_copy_implementations,
+//missing_debug_implementations,
+//missing_docs,
+//must_not_suspend, unstable
+//non_ascii_idents,
+//non_exhaustive_omitted_patterns, unstable
+noop_method_call,
+pointer_structural_match,
+rust_2021_incompatible_closure_captures,
+rust_2021_incompatible_or_patterns,
+//rust_2021_prefixes_incompatible_syntax,
+rust_2021_prelude_collisions,
+single_use_lifetimes,
+trivial_casts,
+trivial_numeric_casts,
+unreachable_pub,
+unsafe_code,
+unsafe_op_in_unsafe_fn,
+stable_features,
+//unused_crate_dependencies
+unused_extern_crates,
+unused_import_braces,
+unused_lifetimes,
+unused_qualifications,
+unused_results,
+variant_size_differences,
+warnings, //treat all warnings as errors
+clippy::all,
+//clippy::pedantic,
+clippy::cargo,
+//The followings are selected restriction lints for rust 1.57
+clippy::as_conversions,
+clippy::clone_on_ref_ptr,
+clippy::create_dir,
+clippy::dbg_macro,
+clippy::decimal_literal_representation,
+//clippy::default_numeric_fallback, too verbose when dealing with numbers
+clippy::disallowed_script_idents,
+clippy::else_if_without_else,
+//clippy::exhaustive_enums,
+clippy::exhaustive_structs,
+clippy::exit,
+clippy::expect_used,
+clippy::filetype_is_file,
+clippy::float_arithmetic,
+clippy::float_cmp_const,
+clippy::get_unwrap,
+clippy::if_then_some_else_none,
+//clippy::implicit_return, it's idiomatic Rust code.
+clippy::indexing_slicing,
+//clippy::inline_asm_x86_att_syntax,
+//clippy::inline_asm_x86_intel_syntax,
+clippy::integer_arithmetic,
+clippy::integer_division,
+clippy::let_underscore_must_use,
+clippy::lossy_float_literal,
+clippy::map_err_ignore,
+clippy::mem_forget,
+//clippy::missing_docs_in_private_items,
+clippy::missing_enforced_import_renames,
+clippy::missing_inline_in_public_items,
+//clippy::mod_module_files, mod.rs file lis used
+clippy::modulo_arithmetic,
+clippy::multiple_inherent_impl,
+//clippy::panic, allow in application code
+//clippy::panic_in_result_fn, not necessary as panic is banned
+clippy::pattern_type_mismatch,
+clippy::print_stderr,
+clippy::print_stdout,
+clippy::rc_buffer,
+clippy::rc_mutex,
+clippy::rest_pat_in_fully_bound_structs,
+clippy::same_name_method,
+clippy::self_named_module_files,
+//clippy::shadow_reuse, it’s a common pattern in Rust code
+//clippy::shadow_same, it’s a common pattern in Rust code
+clippy::shadow_unrelated,
+clippy::str_to_string,
+clippy::string_add,
+clippy::string_to_string,
+clippy::todo,
+clippy::unimplemented,
+clippy::unnecessary_self_imports,
+clippy::unneeded_field_pattern,
+//clippy::unreachable, allow unreachable panic,which is out of expectation
+clippy::unwrap_in_result,
+clippy::unwrap_used,
+//clippy::use_debug, debug is allow for debug log
+clippy::verbose_file_reads,
+clippy::wildcard_enum_match_arm,
+)]
+#[allow(
+clippy::panic, //allow debug_assert,panic in production code
+clippy::multiple_crate_versions, //caused by the dependency, can't be fixed
+)]
 pub mod db;
 pub mod state;
 pub mod store;
diff --git a/storage/src/state/cache.rs b/storage/src/state/cache.rs
@@ -215,9 +215,8 @@ impl SessionedCache {
             return true;
         }
         for delta in self.stack.iter().rev() {
-            match delta.get(key).map(|v| v.is_none()) {
-                Some(v) => return v,
-                None => {}
+            if let Some(v) = delta.get(key).map(|v| v.is_none()) {
+                return v;
             }
         }
         if self.base.get(key) == Some(&None) {