Add br_i{32,64}[_not]_eq_{s,r}z ops

crates/ir/build/display/decode.rs

@@ -163,8 +163,9 @@ impl Display for DisplayDecode<&'_ LoadOp> {
             OperandKind::SlotAndReg => {
                 DisplayMaybe::Some(DisplayConcat((',', FieldTy::SlotAndRegInt)))
             }
-            OperandKind::Immediate => DisplayMaybe::None,
-            OperandKind::Local(_index) => DisplayMaybe::None,
+            OperandKind::Immediate | OperandKind::Zero | OperandKind::Local(_) => {
+                DisplayMaybe::None
+            }
         };
         let generics = DisplayConcat(('<', result_ty, ptr_ty, '>'));
         writeln!(
@@ -198,8 +199,7 @@ impl Display for DisplayDecode<&'_ StoreOp> {
             OperandKind::Reg | OperandKind::Slot | OperandKind::SlotAndReg => {
                 Some(DisplayConcat((op.ptr_field().ty, ',')))
             }
-            OperandKind::Immediate => None,
-            OperandKind::Local(_index) => None,
+            OperandKind::Immediate | OperandKind::Zero | OperandKind::Local(_) => None,
         }
         .display_maybe();
         let value_ty = op.value_field().ty;

crates/ir/build/display/ident.rs

@@ -245,6 +245,7 @@ impl Display for CamelCase<Suffix<OperandKind>> {
             OperandKind::SlotAndReg => "Rs_",
             OperandKind::Reg => "R",
             OperandKind::Immediate => "I",
+            OperandKind::Zero => "Z",
             OperandKind::Local(index) => return write!(f, "S{index}"),
         };
         f.write_str(s)
@@ -258,6 +259,7 @@ impl Display for SnakeCase<Suffix<OperandKind>> {
             OperandKind::SlotAndReg => "rs_",
             OperandKind::Reg => "r",
             OperandKind::Immediate => "i",
+            OperandKind::Zero => "z",
             OperandKind::Local(index) => return write!(f, "s{index}"),
         };
         f.write_str(s)

crates/ir/build/display/result.rs

@@ -170,7 +170,7 @@ impl OperandKind {
             OperandKind::Slot => Some(Location::Slot),
             OperandKind::SlotAndReg => Some(Location::SlotAndReg),
             OperandKind::Local(_) => None,
-            OperandKind::Immediate => unreachable!(),
+            OperandKind::Immediate | OperandKind::Zero => unreachable!(),
         }
     }
 }

crates/ir/build/isa.rs

@@ -372,6 +372,14 @@ fn add_cmp_branch_ops(isa: &mut Isa) {
             }
         }
     }
+    // eqz & nez
+    for ident in [Ident::Eq, Ident::NotEq] {
+        for ty in [Ty::I32, Ty::I64] {
+            for lhs in [OperandKind::Slot, OperandKind::Reg] {
+                isa.push_op(CmpBranchOp::new(ident, ty, lhs, OperandKind::Zero));
+            }
+        }
+    }
 }
 
 fn add_select_ops(isa: &mut Isa) {

crates/ir/build/mod.rs

@@ -77,7 +77,7 @@ pub fn generate_code(config: &Config) -> Result<(), Error> {
 
 fn generate_op_rs(config: &Config, isa: &Isa, contents: &mut String) -> Result<(), Error> {
     let expected_size = match config.simd {
-        true => 505_000,
+        true => 510_000,
         false => 390_000,
     };
     write_to_buffer(contents, expected_size, |buffer| {

crates/ir/build/op.rs

@@ -88,6 +88,8 @@ pub enum OperandKind {
     Local(u16),
     /// The operand is both a slot and a register value.
     SlotAndReg,
+    /// An immediate zero value.
+    Zero,
 }
 
 impl OperandKind {
@@ -108,6 +110,7 @@ impl OperandKind {
                 Ty::F64 | Ty::SignF64 => FieldTy::RegF64,
                 _ => FieldTy::RegInt,
             },
+            OperandKind::Zero => FieldTy::Zero,
             OperandKind::Local(index) => FieldTy::Local(index),
             OperandKind::SlotAndReg => match hint {
                 Ty::F32 | Ty::SignF32 => FieldTy::SlotAndRegF32,
@@ -522,6 +525,7 @@ impl LoadOp {
             OperandKind::Slot => FieldTy::Slot,
             OperandKind::Immediate => FieldTy::Address,
             OperandKind::Reg => FieldTy::RegInt,
+            OperandKind::Zero => FieldTy::Zero,
             OperandKind::Local(index) => FieldTy::Local(index),
             OperandKind::SlotAndReg => FieldTy::SlotAndRegInt,
         };
@@ -625,6 +629,7 @@ impl StoreOp {
             OperandKind::Slot => FieldTy::Slot,
             OperandKind::Reg => FieldTy::RegInt,
             OperandKind::Immediate => FieldTy::Address,
+            OperandKind::Zero => FieldTy::Zero,
             OperandKind::Local(index) => FieldTy::Local(index),
             OperandKind::SlotAndReg => FieldTy::SlotAndRegInt,
         };
@@ -653,6 +658,7 @@ impl StoreOp {
                 StoreKind::Wrap { wrapped } => FieldTy::from(wrapped),
                 StoreKind::Lane { width } => FieldTy::from(width),
             },
+            OperandKind::Zero => FieldTy::Zero,
             OperandKind::Local(index) => FieldTy::Local(index),
         };
         Field::new(Ident::Value, field_ty)
@@ -989,6 +995,7 @@ impl ReplaceLaneOp {
             },
             OperandKind::Immediate => self.ty.item_ty(),
             OperandKind::Local(index) => FieldTy::Local(index),
+            OperandKind::Zero => FieldTy::Zero,
         };
         Field::new(Ident::Value, value_ty)
     }

crates/ir/build/ty.rs

@@ -166,6 +166,7 @@ impl Ty {
 
 #[derive(Copy, Clone)]
 pub enum FieldTy {
+    Zero,
     Slot,
     SlotSpan,
     SlotAndRegInt,
@@ -229,7 +230,7 @@ impl FieldTy {
     pub fn is_unit(&self) -> bool {
         matches!(
             self,
-            Self::Local(_) | Self::Table0 | Self::RegInt | Self::RegF32 | Self::RegF64
+            Self::Local(_) | Self::Table0 | Self::RegInt | Self::RegF32 | Self::RegF64 | Self::Zero
         )
     }
 }
@@ -277,6 +278,7 @@ impl From<Ty> for FieldTy {
 impl Display for FieldTy {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let s = match self {
+            Self::Zero => "Zero",
             Self::Slot => "Slot",
             Self::SlotSpan => "SlotSpan",
             Self::SlotAndRegInt => "SlotAndReg<i64>",

crates/ir/src/decode/mod.rs

@@ -48,6 +48,7 @@ use crate::{
     SlotAndReg,
     SlotSpan,
     Table0,
+    Zero,
     core::{ShiftAmount, TrapCode},
     index::{
         DataAddr,
@@ -260,6 +261,13 @@ impl<const N: u8> Decode for ImmLaneIdx<N> {
     }
 }
 
+impl Decode for Zero {
+    #[inline]
+    fn decode<D: Decoder>(_decoder: &mut D) -> Result<Self, DecodeError> {
+        Ok(Self::default())
+    }
+}
+
 impl<const N: u16> Decode for Local<N> {
     #[inline]
     fn decode<D: Decoder>(_decoder: &mut D) -> Result<Self, DecodeError> {

crates/ir/src/encode.rs

@@ -25,6 +25,7 @@ use crate::{
     SlotSpan,
     Table0,
     TableAddr,
+    Zero,
     core::{ShiftAmount, TrapCode},
     index::RawSlot,
     primitive::OffsetRepr,
@@ -268,6 +269,16 @@ impl<const N: u16> Encode for Local<N> {
     }
 }
 
+impl Encode for Zero {
+    #[inline]
+    fn encode<E>(&self, encoder: &mut E) -> Result<E::Pos, E::Error>
+    where
+        E: Encoder,
+    {
+        encoder.write_bytes(&[])
+    }
+}
+
 impl Encode for Table0 {
     #[inline]
     fn encode<E>(&self, encoder: &mut E) -> Result<E::Pos, E::Error>

crates/ir/src/lib.rs

@@ -45,6 +45,7 @@ pub use self::{
         Reg,
         SlotAndReg,
         Table0,
+        Zero,
     },
     span::{BoundedSlotSpan, FixedSlotSpan, SlotSpan, SlotSpanIter},
 };

crates/ir/src/op.rs

@@ -23,6 +23,7 @@ use crate::{
     SlotAndReg,
     Table0,
     TableAddr,
+    Zero,
     core::{ShiftAmount, TrapCode, ValType},
 };
 use core::num::NonZero;

crates/ir/src/primitive.rs

@@ -5,6 +5,10 @@ use core::{
     marker::PhantomData,
 };
 
+/// An immediate zero value.
+#[derive(Debug, Default, Copy, Clone)]
+pub struct Zero {}
+
 /// Always refers to `(table 0)`.
 #[derive(Debug, Default, Copy, Clone)]
 pub struct Table0 {}

crates/wasmi/src/engine/executor/handler/exec.rs

@@ -1836,8 +1836,10 @@ handler_cmp_branch! {
 
     fn branch_i32_eq_rs(BranchI32Eq_Rs) = wasm::i32_eq;
     fn branch_i32_eq_ri(BranchI32Eq_Ri) = wasm::i32_eq;
+    fn branch_i32_eq_rz(BranchI32Eq_Rz) = wasm::i32_eq;
     fn branch_i32_eq_ss(BranchI32Eq_Ss) = wasm::i32_eq;
     fn branch_i32_eq_si(BranchI32Eq_Si) = wasm::i32_eq;
+    fn branch_i32_eq_sz(BranchI32Eq_Sz) = wasm::i32_eq;
     fn branch_i32_and_rs(BranchI32And_Rs) = eval::wasmi_i32_and;
     fn branch_i32_and_ri(BranchI32And_Ri) = eval::wasmi_i32_and;
     fn branch_i32_and_ss(BranchI32And_Ss) = eval::wasmi_i32_and;
@@ -1848,8 +1850,10 @@ handler_cmp_branch! {
     fn branch_i32_or_si(BranchI32Or_Si) = eval::wasmi_i32_or;
     fn branch_i32_not_eq_rs(BranchI32NotEq_Rs) = wasm::i32_ne;
     fn branch_i32_not_eq_ri(BranchI32NotEq_Ri) = wasm::i32_ne;
+    fn branch_i32_not_eq_rz(BranchI32NotEq_Rz) = wasm::i32_ne;
     fn branch_i32_not_eq_ss(BranchI32NotEq_Ss) = wasm::i32_ne;
     fn branch_i32_not_eq_si(BranchI32NotEq_Si) = wasm::i32_ne;
+    fn branch_i32_not_eq_sz(BranchI32NotEq_Sz) = wasm::i32_ne;
     fn branch_i32_not_and_rs(BranchI32NotAnd_Rs) = eval::wasmi_i32_not_and;
     fn branch_i32_not_and_ri(BranchI32NotAnd_Ri) = eval::wasmi_i32_not_and;
     fn branch_i32_not_and_ss(BranchI32NotAnd_Ss) = eval::wasmi_i32_not_and;
@@ -1895,8 +1899,10 @@ handler_cmp_branch! {
 
     fn branch_i64_eq_rs(BranchI64Eq_Rs) = wasm::i64_eq;
     fn branch_i64_eq_ri(BranchI64Eq_Ri) = wasm::i64_eq;
+    fn branch_i64_eq_rz(BranchI64Eq_Rz) = wasm::i64_eq;
     fn branch_i64_eq_ss(BranchI64Eq_Ss) = wasm::i64_eq;
     fn branch_i64_eq_si(BranchI64Eq_Si) = wasm::i64_eq;
+    fn branch_i64_eq_sz(BranchI64Eq_Sz) = wasm::i64_eq;
     fn branch_i64_and_rs(BranchI64And_Rs) = eval::wasmi_i64_and;
     fn branch_i64_and_ri(BranchI64And_Ri) = eval::wasmi_i64_and;
     fn branch_i64_and_ss(BranchI64And_Ss) = eval::wasmi_i64_and;
@@ -1907,8 +1913,10 @@ handler_cmp_branch! {
     fn branch_i64_or_si(BranchI64Or_Si) = eval::wasmi_i64_or;
     fn branch_i64_not_eq_rs(BranchI64NotEq_Rs) = wasm::i64_ne;
     fn branch_i64_not_eq_ri(BranchI64NotEq_Ri) = wasm::i64_ne;
+    fn branch_i64_not_eq_rz(BranchI64NotEq_Rz) = wasm::i64_ne;
     fn branch_i64_not_eq_ss(BranchI64NotEq_Ss) = wasm::i64_ne;
     fn branch_i64_not_eq_si(BranchI64NotEq_Si) = wasm::i64_ne;
+    fn branch_i64_not_eq_sz(BranchI64NotEq_Sz) = wasm::i64_ne;
     fn branch_i64_not_and_rs(BranchI64NotAnd_Rs) = eval::wasmi_i64_not_and;
     fn branch_i64_not_and_ri(BranchI64NotAnd_Ri) = eval::wasmi_i64_not_and;
     fn branch_i64_not_and_ss(BranchI64NotAnd_Ss) = eval::wasmi_i64_not_and;

crates/wasmi/src/engine/executor/handler/utils.rs

@@ -208,6 +208,20 @@ macro_rules! impl_get_value_for_ireg {
 }
 impl_get_value_for_ireg!(bool, i8, i16, i32, i64, u8, u16, u32, u64, ShiftAmount);
 
+macro_rules! impl_get_value_for_zero {
+    ( $($prim:ty),* $(,)? ) => {
+        $(
+            impl GetValue<$prim> for ir::Zero {
+                #[inline]
+                fn get_value(_src: Self, _sp: Sp, _ireg: Ireg, _freg32: Freg32, _freg64: Freg64) -> $prim {
+                    0
+                }
+            }
+        )*
+    };
+}
+impl_get_value_for_zero!(i8, i16, i32, i64, u8, u16, u32, u64);
+
 impl From<Ireg> for ShiftAmount {
     #[inline]
     fn from(value: Ireg) -> Self {

crates/wasmi/src/engine/translator/comparator.rs

@@ -690,6 +690,34 @@ impl TryIntoCmpBranchInstr for Op {
             | Op::F64NotLe_Ris { lhs, rhs, .. } => Op::branch_f64_not_le_is(offset, lhs, rhs),
             _ => return None,
         };
+        // lower to eqz or nez
+        let cmp_branch_instr = match cmp_branch_instr {
+            | Op::BranchI32Eq_Ri { offset, rhs: 0, .. } => Op::branch_i32_eq_rz(offset),
+            | Op::BranchI32Eq_Si {
+                offset,
+                lhs,
+                rhs: 0,
+            } => Op::branch_i32_eq_sz(offset, lhs),
+            | Op::BranchI32NotEq_Ri { offset, rhs: 0, .. } => Op::branch_i32_not_eq_rz(offset),
+            | Op::BranchI32NotEq_Si {
+                offset,
+                lhs,
+                rhs: 0,
+            } => Op::branch_i32_not_eq_sz(offset, lhs),
+            | Op::BranchI64Eq_Ri { offset, rhs: 0, .. } => Op::branch_i64_eq_rz(offset),
+            | Op::BranchI64Eq_Si {
+                offset,
+                lhs,
+                rhs: 0,
+            } => Op::branch_i64_eq_sz(offset, lhs),
+            | Op::BranchI64NotEq_Ri { offset, rhs: 0, .. } => Op::branch_i64_not_eq_rz(offset),
+            | Op::BranchI64NotEq_Si {
+                offset,
+                lhs,
+                rhs: 0,
+            } => Op::branch_i64_not_eq_sz(offset, lhs),
+            op => op,
+        };
         Some(cmp_branch_instr)
     }
 }
@@ -736,8 +764,10 @@ impl UpdateBranchOffset for Op {
 
             | Op::BranchI32Eq_Rs { offset, .. }
             | Op::BranchI32Eq_Ri { offset, .. }
+            | Op::BranchI32Eq_Rz { offset, .. }
             | Op::BranchI32Eq_Ss { offset, .. }
             | Op::BranchI32Eq_Si { offset, .. }
+            | Op::BranchI32Eq_Sz { offset, .. }
             | Op::BranchI32And_Rs { offset, .. }
             | Op::BranchI32And_Ri { offset, .. }
             | Op::BranchI32And_Ss { offset, .. }
@@ -748,8 +778,10 @@ impl UpdateBranchOffset for Op {
             | Op::BranchI32Or_Si { offset, .. }
             | Op::BranchI32NotEq_Rs { offset, .. }
             | Op::BranchI32NotEq_Ri { offset, .. }
+            | Op::BranchI32NotEq_Rz { offset, .. }
             | Op::BranchI32NotEq_Ss { offset, .. }
             | Op::BranchI32NotEq_Si { offset, .. }
+            | Op::BranchI32NotEq_Sz { offset, .. }
             | Op::BranchI32NotAnd_Rs { offset, .. }
             | Op::BranchI32NotAnd_Ri { offset, .. }
             | Op::BranchI32NotAnd_Ss { offset, .. }
@@ -795,8 +827,10 @@ impl UpdateBranchOffset for Op {
 
             | Op::BranchI64Eq_Rs { offset, .. }
             | Op::BranchI64Eq_Ri { offset, .. }
+            | Op::BranchI64Eq_Rz { offset, .. }
             | Op::BranchI64Eq_Ss { offset, .. }
             | Op::BranchI64Eq_Si { offset, .. }
+            | Op::BranchI64Eq_Sz { offset, .. }
             | Op::BranchI64And_Rs { offset, .. }
             | Op::BranchI64And_Ri { offset, .. }
             | Op::BranchI64And_Ss { offset, .. }
@@ -807,8 +841,10 @@ impl UpdateBranchOffset for Op {
             | Op::BranchI64Or_Si { offset, .. }
             | Op::BranchI64NotEq_Rs { offset, .. }
             | Op::BranchI64NotEq_Ri { offset, .. }
+            | Op::BranchI64NotEq_Rz { offset, .. }
             | Op::BranchI64NotEq_Ss { offset, .. }
             | Op::BranchI64NotEq_Si { offset, .. }
+            | Op::BranchI64NotEq_Sz { offset, .. }
             | Op::BranchI64NotAnd_Rs { offset, .. }
             | Op::BranchI64NotAnd_Ri { offset, .. }
             | Op::BranchI64NotAnd_Ss { offset, .. }

crates/wasmi/src/engine/translator/func/mod.rs

@@ -1548,12 +1548,12 @@ impl FuncTranslator {
         };
         self.encode_branch_op(label, |offset| match branch_eqz {
             true => match condition {
-                Location::Slot(condition) => Op::branch_i32_eq_si(offset, condition, 0),
-                Location::Reg(_) => Op::branch_i32_eq_ri(offset, 0),
+                Location::Slot(condition) => Op::branch_i32_eq_sz(offset, condition),
+                Location::Reg(_) => Op::branch_i32_eq_rz(offset),
             },
             false => match condition {
-                Location::Slot(condition) => Op::branch_i32_not_eq_si(offset, condition, 0),
-                Location::Reg(_) => Op::branch_i32_not_eq_ri(offset, 0),
+                Location::Slot(condition) => Op::branch_i32_not_eq_sz(offset, condition),
+                Location::Reg(_) => Op::branch_i32_not_eq_rz(offset),
             },
         })?;
         Ok(())

crates/wasmi/src/engine/translator/func/visit.rs

@@ -192,8 +192,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator {
                 self.encode_branch_op(else_label, |offset| match fused_op {
                     Some(fused_op) => fused_op.with_branch_offset(offset),
                     None => match condition {
-                        Location::Reg(_) => Op::branch_i32_eq_ri(offset, 0),
-                        Location::Slot(condition) => Op::branch_i32_eq_si(offset, condition, 0),
+                        Location::Reg(_) => Op::branch_i32_eq_rz(offset),
+                        Location::Slot(condition) => Op::branch_i32_eq_sz(offset, condition),
                     },
                 })?;
                 let reachability = IfReachability::Both { else_label };