Optimize Wasmi executor (#1206)

* add inline and cold attributes to load/store instr impls * add inline(never) to costly instruction impls * remove inline(always) annotations for execution handlers * move fetch_optional_memory up
wasmi-labs · Sep 29, 2024 · fd8dbd1 · fd8dbd1
1 parent 9f46524
commit fd8dbd1
Show file tree

Hide file tree

Showing 14 changed files with 69 additions and 218 deletions.
diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs
@@ -2,7 +2,7 @@ pub use self::call::{dispatch_host_func, ResumableHostError};
 use self::return_::ReturnOutcome;
 use super::{cache::CachedInstance, InstructionPtr, Stack};
 use crate::{
-    core::{TrapCode, UntypedVal},
+    core::{hint, TrapCode, UntypedVal},
     engine::{
         bytecode::{index, BlockFuel, Const16, Instruction, Reg},
         code_map::CodeMap,
@@ -1684,6 +1684,7 @@ impl<'engine> Executor<'engine> {
     }
 
     /// Skips all [`Instruction`]s belonging to an [`Instruction::RegisterList`] encoding.
+    #[inline(always)]
     fn skip_register_list(ip: InstructionPtr) -> InstructionPtr {
         let mut ip = ip;
         while let Instruction::RegisterList { .. } = *ip.get() {
@@ -1696,6 +1697,26 @@ impl<'engine> Executor<'engine> {
         ip.add(1);
         ip
     }
+
+    /// Returns the optional `memory` parameter for a `load_at` [`Instruction`].
+    ///
+    /// # Note
+    ///
+    /// - Returns the default [`index::Memory`] if the parameter is missing.
+    /// - Bumps `self.ip` if a [`Instruction::MemoryIndex`] parameter was found.
+    #[inline(always)]
+    fn fetch_optional_memory(&mut self) -> index::Memory {
+        let mut addr: InstructionPtr = self.ip;
+        addr.add(1);
+        match *addr.get() {
+            Instruction::MemoryIndex { index } => {
+                hint::cold();
+                self.ip = addr;
+                index
+            }
+            _ => index::Memory::from(0),
+        }
+    }
 }
 
 impl<'engine> Executor<'engine> {
@@ -1705,19 +1726,16 @@ impl<'engine> Executor<'engine> {
     ///
     /// This includes [`Instruction`] variants such as [`Instruction::TableIndex`]
     /// that primarily carry parameters for actually executable [`Instruction`].
-    #[inline(always)]
     fn invalid_instruction_word(&mut self) -> Result<(), Error> {
         self.execute_trap(TrapCode::UnreachableCodeReached)
     }
 
     /// Executes a Wasm `unreachable` instruction.
-    #[inline(always)]
     fn execute_trap(&mut self, trap_code: TrapCode) -> Result<(), Error> {
         Err(Error::from(trap_code))
     }
 
     /// Executes an [`Instruction::ConsumeFuel`].
-    #[inline(always)]
     fn execute_consume_fuel(
         &mut self,
         store: &mut StoreInner,
@@ -1733,32 +1751,12 @@ impl<'engine> Executor<'engine> {
     }
 
     /// Executes an [`Instruction::RefFunc`].
-    #[inline(always)]
     fn execute_ref_func(&mut self, result: Reg, func_index: index::Func) {
         let func = self.get_func(func_index);
         let funcref = FuncRef::new(func);
         self.set_register(result, funcref);
         self.next_instr();
     }
-
-    /// Returns the optional `memory` parameter for a `load_at` [`Instruction`].
-    ///
-    /// # Note
-    ///
-    /// - Returns the default [`index::Memory`] if the parameter is missing.
-    /// - Bumps `self.ip` if a [`Instruction::MemoryIndex`] parameter was found.
-    #[inline]
-    fn fetch_optional_memory(&mut self) -> index::Memory {
-        let mut addr: InstructionPtr = self.ip;
-        addr.add(1);
-        match *addr.get() {
-            Instruction::MemoryIndex { index } => {
-                self.ip = addr;
-                index
-            }
-            _ => index::Memory::from(0),
-        }
-    }
 }
 
 /// Extension method for [`UntypedVal`] required by the [`Executor`].

diff --git a/crates/wasmi/src/engine/executor/instrs/binary.rs b/crates/wasmi/src/engine/executor/instrs/binary.rs
@@ -13,7 +13,6 @@ macro_rules! impl_binary {
     ( $( (Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Reg) {
                 self.execute_binary(result, lhs, rhs, $op)
             }
@@ -73,7 +72,6 @@ macro_rules! impl_binary_imm16 {
     ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Const16<$ty>) {
                 self.execute_binary_imm16(result, lhs, rhs, $op)
             }
@@ -116,7 +114,6 @@ macro_rules! impl_binary_imm16_rev {
     ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Const16<$ty>, rhs: Reg) {
                 self.execute_binary_imm16_rev(result, lhs, rhs, $op)
             }
@@ -146,7 +143,6 @@ macro_rules! impl_fallible_binary {
     ( $( (Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Reg) -> Result<(), Error> {
                 self.try_execute_binary(result, lhs, rhs, $op).map_err(Into::into)
             }
@@ -238,7 +234,6 @@ macro_rules! impl_divrem_s_imm16 {
     ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Const16<$ty>) -> Result<(), Error> {
                 self.try_execute_divrem_imm16(result, lhs, rhs, $op)
             }
@@ -259,7 +254,6 @@ macro_rules! impl_divrem_u_imm16 {
     ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Const16<$ty>) {
                 self.execute_divrem_imm16(result, lhs, rhs, $op)
             }
@@ -280,7 +274,6 @@ macro_rules! impl_fallible_binary_imm16_rev {
     ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => {
         $(
             #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")]
-            #[inline(always)]
             pub fn $fn_name(&mut self, result: Reg, lhs: Const16<$ty>, rhs: Reg) -> Result<(), Error> {
                 self.try_execute_binary_imm16_rev(result, lhs, rhs, $op).map_err(Into::into)
             }
@@ -303,7 +296,6 @@ impl<'engine> Executor<'engine> {
 
 impl<'engine> Executor<'engine> {
     /// Executes an [`Instruction::F32CopysignImm`].
-    #[inline(always)]
     pub fn execute_f32_copysign_imm(&mut self, result: Reg, lhs: Reg, rhs: Sign<f32>) {
         let lhs = self.get_register(lhs);
         let rhs = f32::from(rhs);
@@ -312,7 +304,6 @@ impl<'engine> Executor<'engine> {
     }
 
     /// Executes an [`Instruction::F64CopysignImm`].
-    #[inline(always)]
     pub fn execute_f64_copysign_imm(&mut self, result: Reg, lhs: Reg, rhs: Sign<f64>) {
         let lhs = self.get_register(lhs);
         let rhs = f64::from(rhs);

diff --git a/crates/wasmi/src/engine/executor/instrs/branch.rs b/crates/wasmi/src/engine/executor/instrs/branch.rs
@@ -19,7 +19,6 @@ impl<'engine> Executor<'engine> {
     /// # Note
     ///
     /// Offsets the instruction pointer using the given [`BranchOffset`].
-    #[inline(always)]
     fn branch_to(&mut self, offset: BranchOffset) {
         self.ip.offset(offset.to_i32() as isize)
     }
@@ -29,12 +28,10 @@ impl<'engine> Executor<'engine> {
     /// # Note
     ///
     /// Offsets the instruction pointer using the given [`BranchOffset`].
-    #[inline(always)]
     fn branch_to16(&mut self, offset: BranchOffset16) {
         self.ip.offset(offset.to_i16() as isize)
     }
 
-    #[inline(always)]
     pub fn execute_branch(&mut self, offset: BranchOffset) {
         self.branch_to(offset)
     }
@@ -48,13 +45,11 @@ impl<'engine> Executor<'engine> {
         cmp::min(index, max_index) as usize + 1
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_0(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets);
         self.ip.add(offset);
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_1(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets);
         self.ip.add(1);
@@ -74,7 +69,6 @@ impl<'engine> Executor<'engine> {
         }
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_2(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets);
         self.ip.add(1);
@@ -94,7 +88,6 @@ impl<'engine> Executor<'engine> {
         }
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_3(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets);
         self.ip.add(1);
@@ -114,7 +107,6 @@ impl<'engine> Executor<'engine> {
         }
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_span(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets);
         self.ip.add(1);
@@ -139,7 +131,6 @@ impl<'engine> Executor<'engine> {
         }
     }
 
-    #[inline(always)]
     pub fn execute_branch_table_many(&mut self, index: Reg, len_targets: u32) {
         let offset = self.fetch_branch_table_offset(index, len_targets) - 1;
         self.ip.add(1);
@@ -168,7 +159,6 @@ impl<'engine> Executor<'engine> {
     }
 
     /// Executes a generic fused compare and branch instruction with raw inputs.
-    #[inline(always)]
     fn execute_branch_binop<T>(
         &mut self,
         lhs: Reg,
@@ -187,7 +177,6 @@ impl<'engine> Executor<'engine> {
     }
 
     /// Executes a generic fused compare and branch instruction with immediate `rhs` operand.
-    #[inline(always)]
     fn execute_branch_binop_imm<T>(
         &mut self,
         lhs: Reg,
@@ -277,7 +266,6 @@ macro_rules! impl_execute_branch_binop {
         impl<'engine> Executor<'engine> {
             $(
                 #[doc = concat!("Executes an [`Instruction::", stringify!($op_name), "`].")]
-                #[inline(always)]
                 pub fn $fn_name(&mut self, lhs: Reg, rhs: Reg, offset: BranchOffset16) {
                     self.execute_branch_binop::<$ty>(lhs, rhs, offset, $op)
                 }
@@ -334,7 +322,6 @@ macro_rules! impl_execute_branch_binop_imm {
         impl<'engine> Executor<'engine> {
             $(
                 #[doc = concat!("Executes an [`Instruction::", stringify!($op_name), "`].")]
-                #[inline(always)]
                 pub fn $fn_name(&mut self, lhs: Reg, rhs: Const16<$ty>, offset: BranchOffset16) {
                     self.execute_branch_binop_imm::<$ty>(lhs, rhs, offset, $op)
                 }