From e4b0b71164ba33caf41d0d76838a0cda1a9ac0c7 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Mon, 20 Jun 2022 10:49:36 +0200 Subject: [PATCH] Converted all machine and emitter of singlepass to use Result instead of panic (but assert are still present) --- lib/compiler-singlepass/src/codegen.rs | 2163 ++++----- lib/compiler-singlepass/src/compiler.rs | 16 +- lib/compiler-singlepass/src/emitter_arm64.rs | 1462 ++++-- lib/compiler-singlepass/src/emitter_x64.rs | 1146 +++-- lib/compiler-singlepass/src/machine.rs | 574 +-- lib/compiler-singlepass/src/machine_arm64.rs | 3064 +++++++----- lib/compiler-singlepass/src/machine_x64.rs | 4391 +++++++++++------- 7 files changed, 7811 insertions(+), 5005 deletions(-) diff --git a/lib/compiler-singlepass/src/codegen.rs b/lib/compiler-singlepass/src/codegen.rs index d7836e55a76..dd0a536e4f8 100644 --- a/lib/compiler-singlepass/src/codegen.rs +++ b/lib/compiler-singlepass/src/codegen.rs @@ -1,4 +1,5 @@ use crate::address_map::get_function_address_map; +use crate::codegen_error; #[cfg(feature = "unwind")] use crate::dwarf::WriterRelocate; use crate::location::{Location, Reg}; @@ -126,26 +127,28 @@ impl FloatValue { } } - fn promote(self, depth: usize) -> FloatValue { - FloatValue { + fn promote(self, depth: usize) -> Result { + let ret = FloatValue { canonicalization: match self.canonicalization { Some(CanonicalizeType::F32) => Some(CanonicalizeType::F64), - Some(CanonicalizeType::F64) => panic!("cannot promote F64"), + Some(CanonicalizeType::F64) => codegen_error!("cannot promote F64"), None => None, }, depth, - } + }; + Ok(ret) } - fn demote(self, depth: usize) -> FloatValue { - FloatValue { + fn demote(self, depth: usize) -> Result { + let ret = FloatValue { canonicalization: match self.canonicalization { Some(CanonicalizeType::F64) => Some(CanonicalizeType::F32), - Some(CanonicalizeType::F32) => panic!("cannot demote F32"), + Some(CanonicalizeType::F32) => codegen_error!("cannot demote F32"), None => None, }, depth, - } + }; + Ok(ret) } } @@ -254,11 +257,12 @@ impl<'a, M: Machine> FuncGen<'a, M> { /// /// If the returned locations are used for stack value, `release_location` needs to be called on them; /// Otherwise, if the returned locations are used for locals, `release_location` does not need to be called on them. + #[allow(clippy::type_complexity)] fn acquire_locations( &mut self, tys: &[(WpType, MachineValue)], zeroed: bool, - ) -> SmallVec<[Location; 1]> { + ) -> Result; 1]>, CodegenError> { let mut ret = smallvec![]; let mut delta_stack_offset: usize = 0; @@ -267,7 +271,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { WpType::F32 | WpType::F64 => self.machine.pick_simd().map(Location::SIMD), WpType::I32 | WpType::I64 => self.machine.pick_gpr().map(Location::GPR), WpType::FuncRef | WpType::ExternRef => self.machine.pick_gpr().map(Location::GPR), - _ => unreachable!("can't acquire location for type {:?}", ty), + _ => codegen_error!("can't acquire location for type {:?}", ty), }; let loc = if let Some(x) = loc { @@ -292,18 +296,21 @@ impl<'a, M: Machine> FuncGen<'a, M> { let delta_stack_offset = self.machine.round_stack_adjust(delta_stack_offset); if delta_stack_offset != 0 { - self.machine.adjust_stack(delta_stack_offset as u32); + self.machine.adjust_stack(delta_stack_offset as u32)?; } if zeroed { for i in 0..tys.len() { - self.machine.zero_location(Size::S64, ret[i]); + self.machine.zero_location(Size::S64, ret[i])?; } } - ret + Ok(ret) } /// Releases locations used for stack value. - fn release_locations(&mut self, locs: &[Location]) { + fn release_locations( + &mut self, + locs: &[Location], + ) -> Result<(), CodegenError> { let mut delta_stack_offset: usize = 0; for loc in locs.iter().rev() { @@ -321,28 +328,37 @@ impl<'a, M: Machine> FuncGen<'a, M> { Location::Memory(y, x) => { if y == self.machine.local_pointer() { if x >= 0 { - unreachable!(); + codegen_error!("Invalid memory offset {}", x); } let offset = (-x) as usize; if offset != self.stack_offset.0 { - unreachable!(); + codegen_error!( + "Invalid memory offset {}!={}", + offset, + self.stack_offset.0 + ); } self.stack_offset.0 -= 8; delta_stack_offset += 8; - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Empty stack_value".to_string(), + })?; } } _ => {} } - self.state.wasm_stack.pop().unwrap(); + self.state.wasm_stack.pop().ok_or(CodegenError { + message: "Pop with wasm stack empty".to_string(), + })?; } let delta_stack_offset = self.machine.round_stack_adjust(delta_stack_offset); if delta_stack_offset != 0 { - self.machine.restore_stack(delta_stack_offset as u32); + self.machine.restore_stack(delta_stack_offset as u32)?; } + Ok(()) } /// Releases locations used for stack value. - fn release_locations_value(&mut self, stack_depth: usize) { + fn release_locations_value(&mut self, stack_depth: usize) -> Result<(), CodegenError> { let mut delta_stack_offset: usize = 0; let locs: &[Location] = &self.value_stack[stack_depth..]; @@ -361,29 +377,41 @@ impl<'a, M: Machine> FuncGen<'a, M> { Location::Memory(y, x) => { if y == self.machine.local_pointer() { if x >= 0 { - unreachable!(); + codegen_error!("Invalid memory offset {}", x); } let offset = (-x) as usize; if offset != self.stack_offset.0 { - unreachable!(); + codegen_error!( + "Invalid memory offset {}!={}", + offset, + self.stack_offset.0 + ); } self.stack_offset.0 -= 8; delta_stack_offset += 8; - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop with values stack empty".to_string(), + })?; } } _ => {} } - self.state.wasm_stack.pop().unwrap(); + self.state.wasm_stack.pop().ok_or(CodegenError { + message: "Pop with wasm stack empty".to_string(), + })?; } let delta_stack_offset = self.machine.round_stack_adjust(delta_stack_offset); if delta_stack_offset != 0 { - self.machine.adjust_stack(delta_stack_offset as u32); + self.machine.adjust_stack(delta_stack_offset as u32)?; } + Ok(()) } - fn release_locations_only_regs(&mut self, locs: &[Location]) { + fn release_locations_only_regs( + &mut self, + locs: &[Location], + ) -> Result<(), CodegenError> { for loc in locs.iter().rev() { match *loc { Location::GPR(ref x) => { @@ -400,24 +428,30 @@ impl<'a, M: Machine> FuncGen<'a, M> { } // Wasm state popping is deferred to `release_locations_only_osr_state`. } + Ok(()) } - fn release_locations_only_stack(&mut self, locs: &[Location]) { + fn release_locations_only_stack( + &mut self, + locs: &[Location], + ) -> Result<(), CodegenError> { let mut delta_stack_offset: usize = 0; for loc in locs.iter().rev() { if let Location::Memory(y, x) = *loc { if y == self.machine.local_pointer() { if x >= 0 { - unreachable!(); + codegen_error!("Invalid memory offset {}", x); } let offset = (-x) as usize; if offset != self.stack_offset.0 { - unreachable!(); + codegen_error!("Invalid memory offset {}!={}", offset, self.stack_offset.0); } self.stack_offset.0 -= 8; delta_stack_offset += 8; - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop on empty value stack".to_string(), + })?; } } // Wasm state popping is deferred to `release_locations_only_osr_state`. @@ -425,11 +459,12 @@ impl<'a, M: Machine> FuncGen<'a, M> { let delta_stack_offset = self.machine.round_stack_adjust(delta_stack_offset); if delta_stack_offset != 0 { - self.machine.pop_stack_locals(delta_stack_offset as u32); + self.machine.pop_stack_locals(delta_stack_offset as u32)?; } + Ok(()) } - fn release_locations_only_osr_state(&mut self, n: usize) { + fn release_locations_only_osr_state(&mut self, n: usize) -> Result<(), CodegenError> { let new_length = self .state .wasm_stack @@ -437,9 +472,10 @@ impl<'a, M: Machine> FuncGen<'a, M> { .checked_sub(n) .expect("release_locations_only_osr_state: length underflow"); self.state.wasm_stack.truncate(new_length); + Ok(()) } - fn release_locations_keep_state(&mut self, stack_depth: usize) { + fn release_locations_keep_state(&mut self, stack_depth: usize) -> Result<(), CodegenError> { let mut delta_stack_offset: usize = 0; let mut stack_offset = self.stack_offset.0; let locs = &self.value_stack[stack_depth..]; @@ -448,11 +484,11 @@ impl<'a, M: Machine> FuncGen<'a, M> { if let Location::Memory(y, x) = *loc { if y == self.machine.local_pointer() { if x >= 0 { - unreachable!(); + codegen_error!("Invalid memory offset {}", x); } let offset = (-x) as usize; if offset != stack_offset { - unreachable!(); + codegen_error!("Invalid memory offset {}!={}", offset, self.stack_offset.0); } stack_offset -= 8; delta_stack_offset += 8; @@ -462,16 +498,18 @@ impl<'a, M: Machine> FuncGen<'a, M> { let delta_stack_offset = self.machine.round_stack_adjust(delta_stack_offset); if delta_stack_offset != 0 { - self.machine.pop_stack_locals(delta_stack_offset as u32); + self.machine.pop_stack_locals(delta_stack_offset as u32)?; } + Ok(()) } + #[allow(clippy::type_complexity)] fn init_locals( &mut self, n: usize, sig: FunctionType, calling_convention: CallingConvention, - ) -> Vec> { + ) -> Result>, CodegenError> { // How many machine stack slots will all the locals use? let num_mem_slots = (0..n) .filter(|&x| self.machine.is_local_on_stack(x)) @@ -518,16 +556,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { .step_by(NATIVE_PAGE_SIZE / 8) .skip(1) { - self.machine.zero_location(Size::S64, locations[i]); + self.machine.zero_location(Size::S64, locations[i])?; } - self.machine.adjust_stack(static_area_size as _); + self.machine.adjust_stack(static_area_size as _)?; // Save callee-saved registers. for loc in locations.iter() { if let Location::GPR(x) = *loc { self.stack_offset.0 += 8; - self.machine.move_local(self.stack_offset.0 as i32, *loc); + self.machine.move_local(self.stack_offset.0 as i32, *loc)?; self.state.stack_values.push(MachineValue::PreserveRegister( self.machine.index_from_gpr(x), )); @@ -539,7 +577,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.machine.move_local( self.stack_offset.0 as i32, Location::GPR(self.machine.get_vmctx_reg()), - ); + )?; self.state.stack_values.push(MachineValue::PreserveRegister( self.machine.index_from_gpr(self.machine.get_vmctx_reg()), )); @@ -548,7 +586,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { let regs_to_save = self.machine.list_to_save(calling_convention); for loc in regs_to_save.iter() { self.stack_offset.0 += 8; - self.machine.move_local(self.stack_offset.0 as i32, *loc); + self.machine.move_local(self.stack_offset.0 as i32, *loc)?; } // Save the offset of register save area. @@ -564,7 +602,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Location::Memory(_, _) => { self.state.stack_values.push(MachineValue::WasmLocal(i)); } - _ => unreachable!(), + _ => codegen_error!("singlpass init_local unreachable"), } } @@ -577,7 +615,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Type::I32 | Type::F32 => Size::S32, Type::I64 | Type::F64 => Size::S64, Type::ExternRef | Type::FuncRef => Size::S64, - _ => unimplemented!(), + _ => codegen_error!("singlepass init_local unimplemented"), }; let loc = self.machine.get_call_param_location( i + 1, @@ -586,7 +624,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { calling_convention, ); self.machine - .move_location_extend(sz, false, loc, Size::S64, locations[i]); + .move_location_extend(sz, false, loc, Size::S64, locations[i])?; } // Load vmctx into it's GPR. @@ -595,7 +633,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.machine .get_simple_param_location(0, calling_convention), Location::GPR(self.machine.get_vmctx_reg()), - ); + )?; // Initialize all normal locals to zero. let mut init_stack_loc_cnt = 0; @@ -607,42 +645,46 @@ impl<'a, M: Machine> FuncGen<'a, M> { last_stack_loc = cmp::min(last_stack_loc, *location); } Location::GPR(_) => { - self.machine.zero_location(Size::S64, *location); + self.machine.zero_location(Size::S64, *location)?; } - _ => unreachable!(), + _ => codegen_error!("singlepass init_local unreachable"), } } if init_stack_loc_cnt > 0 { self.machine - .init_stack_loc(init_stack_loc_cnt, last_stack_loc); + .init_stack_loc(init_stack_loc_cnt, last_stack_loc)?; } // Add the size of all locals allocated to stack. self.stack_offset.0 += static_area_size - callee_saved_regs_size; - locations + Ok(locations) } - fn finalize_locals(&mut self, calling_convention: CallingConvention) { + fn finalize_locals( + &mut self, + calling_convention: CallingConvention, + ) -> Result<(), CodegenError> { // Unwind stack to the "save area". self.machine - .restore_saved_area(self.save_area_offset.as_ref().unwrap().0 as i32); + .restore_saved_area(self.save_area_offset.as_ref().unwrap().0 as i32)?; let regs_to_save = self.machine.list_to_save(calling_convention); for loc in regs_to_save.iter().rev() { - self.machine.pop_location(*loc); + self.machine.pop_location(*loc)?; } // Restore register used by vmctx. self.machine - .pop_location(Location::GPR(self.machine.get_vmctx_reg())); + .pop_location(Location::GPR(self.machine.get_vmctx_reg()))?; // Restore callee-saved registers. for loc in self.locals.iter().rev() { if let Location::GPR(_) = *loc { - self.machine.pop_location(*loc); + self.machine.pop_location(*loc)?; } } + Ok(()) } /// Set the source location of the Wasm to the given offset. @@ -653,29 +695,28 @@ impl<'a, M: Machine> FuncGen<'a, M> { fn get_location_released( &mut self, loc: Location, - ) -> Location { - self.release_locations(&[loc]); - loc + ) -> Result, CodegenError> { + self.release_locations(&[loc])?; + Ok(loc) } - fn pop_value_released(&mut self) -> Location { - let loc = self - .value_stack - .pop() - .expect("pop_value_released: value stack is empty"); + fn pop_value_released(&mut self) -> Result, CodegenError> { + let loc = self.value_stack.pop().ok_or(CodegenError { + message: "pop_value_released: value stack is empty".to_string(), + })?; self.get_location_released(loc) } /// Prepare data for binary operator with 2 inputs and 1 output. - fn i2o1_prepare(&mut self, ty: WpType) -> I2O1 { - let loc_b = self.pop_value_released(); - let loc_a = self.pop_value_released(); + fn i2o1_prepare(&mut self, ty: WpType) -> Result, CodegenError> { + let loc_b = self.pop_value_released()?; + let loc_a = self.pop_value_released()?; let ret = self.acquire_locations( &[(ty, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - I2O1 { loc_a, loc_b, ret } + Ok(I2O1 { loc_a, loc_b, ret }) } fn mark_trappable(&mut self) { @@ -717,7 +758,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { fn emit_call_native< I: Iterator>, J: Iterator, - F: FnOnce(&mut Self), + F: FnOnce(&mut Self) -> Result<(), CodegenError>, >( &mut self, cb: F, @@ -738,7 +779,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { // Save used GPRs. Preserve correct stack alignment let used_gprs = self.machine.get_used_gprs(); - let mut used_stack = self.machine.push_used_gpr(&used_gprs); + let mut used_stack = self.machine.push_used_gpr(&used_gprs)?; for r in used_gprs.iter() { let content = self.state.register_values[self.machine.index_from_gpr(*r).0].clone(); if content == MachineValue::Undefined { @@ -752,7 +793,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { // Save used SIMD registers. let used_simds = self.machine.get_used_simd(); if !used_simds.is_empty() { - used_stack += self.machine.push_used_simd(&used_simds); + used_stack += self.machine.push_used_simd(&used_simds)?; for r in used_simds.iter().rev() { let content = @@ -796,7 +837,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { if stack_unaligned != 0 { stack_offset += 16 - stack_unaligned; } - self.machine.adjust_stack(stack_offset as u32); + self.machine.adjust_stack(stack_offset as u32)?; #[allow(clippy::type_complexity)] let mut call_movs: Vec<(Location, M::GPR)> = vec![]; @@ -844,7 +885,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { } } self.machine - .move_location_for_native(params_size[i], *param, loc); + .move_location_for_native(params_size[i], *param, loc)?; } _ => { return Err(CodegenError { @@ -861,7 +902,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { for (loc, gpr) in call_movs { if loc != Location::GPR(gpr) { self.machine - .move_location(Size::S64, loc, Location::GPR(gpr)); + .move_location(Size::S64, loc, Location::GPR(gpr))?; } } @@ -871,14 +912,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { Location::GPR(self.machine.get_vmctx_reg()), self.machine .get_simple_param_location(0, calling_convention), - ); // vmctx + )?; // vmctx if stack_padding > 0 { - self.machine.adjust_stack(stack_padding as u32); + self.machine.adjust_stack(stack_padding as u32)?; } // release the GPR used for call self.machine.release_gpr(self.machine.get_grp_for_call()); - cb(self); + cb(self)?; // Offset needs to be after the 'call' instruction. // TODO: Now the state information is also inserted for internal calls (e.g. MemoryGrow). Is this expected? @@ -903,32 +944,41 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.machine.restore_stack( self.machine .round_stack_adjust(stack_offset + stack_padding) as u32, - ); + )?; if (stack_offset % 8) != 0 { return Err(CodegenError { message: "emit_call_native: Bad restoring stack alignement".to_string(), }); } for _ in 0..pushed_args { - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop an empty value stack".to_string(), + })?; } } // Restore SIMDs. if !used_simds.is_empty() { - self.machine.pop_used_simd(&used_simds); + self.machine.pop_used_simd(&used_simds)?; for _ in 0..used_simds.len() { - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop an empty value stack".to_string(), + })?; } } // Restore GPRs. - self.machine.pop_used_gpr(&used_gprs); + self.machine.pop_used_gpr(&used_gprs)?; for _ in used_gprs.iter().rev() { - self.state.stack_values.pop().unwrap(); + self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop an empty value stack".to_string(), + })?; } - if self.state.stack_values.pop().unwrap() != MachineValue::ExplicitShadow { + if self.state.stack_values.pop().ok_or(CodegenError { + message: "Pop an empty value stack".to_string(), + })? != MachineValue::ExplicitShadow + { return Err(CodegenError { message: "emit_call_native: Popped value is not ExplicitShadow".to_string(), }); @@ -955,7 +1005,10 @@ impl<'a, M: Machine> FuncGen<'a, M> { } /// Emits a memory operation. - fn op_memory(&mut self, cb: F) { + fn op_memory Result<(), CodegenError>>( + &mut self, + cb: F, + ) -> Result<(), CodegenError> { let need_check = match self.memory_styles[MemoryIndex::new(0)] { MemoryStyle::Static { .. } => false, MemoryStyle::Dynamic { .. } => true, @@ -974,7 +1027,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.module.num_imported_memories != 0, offset as i32, self.special_labels.heap_access_oob, - ); + ) } pub fn get_state_diff(&mut self) -> usize { @@ -992,14 +1045,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { } fn emit_head(&mut self) -> Result<(), CodegenError> { - self.machine.emit_function_prolog(); + self.machine.emit_function_prolog()?; // Initialize locals. self.locals = self.init_locals( self.local_types.len(), self.signature.clone(), self.calling_convention, - ); + )?; // Mark vmctx register. The actual loading of the vmctx value is handled by init_local. self.state.register_values[self.machine.index_from_gpr(self.machine.get_vmctx_reg()).0] = @@ -1011,7 +1064,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.fsm.diffs.push(diff); // simulate "red zone" if not supported by the platform - self.machine.adjust_stack(32); + self.machine.adjust_stack(32)?; self.control_stack.push(ControlFrame { label: self.machine.get_label(), @@ -1164,7 +1217,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { let loc = self.acquire_locations( &[(ty, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(loc); let tmp = self.machine.acquire_temp_gpr().unwrap(); @@ -1177,7 +1230,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), offset as i32), Location::GPR(tmp), - ); + )?; Location::Memory(tmp, 0) } else { // Imported globals require one level of indirection. @@ -1188,11 +1241,11 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), offset as i32), Location::GPR(tmp), - ); + )?; Location::Memory(tmp, 0) }; - self.machine.emit_relaxed_mov(Size::S64, src, loc); + self.machine.emit_relaxed_mov(Size::S64, src, loc)?; self.machine.release_gpr(tmp); } @@ -1207,7 +1260,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), offset as i32), Location::GPR(tmp), - ); + )?; Location::Memory(tmp, 0) } else { // Imported globals require one level of indirection. @@ -1218,11 +1271,11 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), offset as i32), Location::GPR(tmp), - ); + )?; Location::Memory(tmp, 0) }; let ty = type_to_wp_type(self.module.globals[global_index].ty); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; if ty.is_float() { let fp = self.fp_stack.pop1()?; if self.machine.arch_supports_canonicalize_nan() @@ -1233,16 +1286,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { match ty { WpType::F32 => Size::S32, WpType::F64 => Size::S64, - _ => unreachable!(), + _ => codegen_error!("singlepass Operator::GlobalSet unreachable"), }, loc, dst, - ); + )?; } else { - self.machine.emit_relaxed_mov(Size::S64, loc, dst); + self.machine.emit_relaxed_mov(Size::S64, loc, dst)?; } } else { - self.machine.emit_relaxed_mov(Size::S64, loc, dst); + self.machine.emit_relaxed_mov(Size::S64, loc, dst)?; } self.machine.release_gpr(tmp); } @@ -1251,9 +1304,9 @@ impl<'a, M: Machine> FuncGen<'a, M> { let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.machine - .emit_relaxed_mov(Size::S64, self.locals[local_index], ret); + .emit_relaxed_mov(Size::S64, self.locals[local_index], ret)?; self.value_stack.push(ret); if self.local_types[local_index].is_float() { self.fp_stack @@ -1262,7 +1315,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { } Operator::LocalSet { local_index } => { let local_index = local_index as usize; - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; if self.local_types[local_index].is_float() { let fp = self.fp_stack.pop1()?; @@ -1274,19 +1327,19 @@ impl<'a, M: Machine> FuncGen<'a, M> { match self.local_types[local_index] { WpType::F32 => Size::S32, WpType::F64 => Size::S64, - _ => unreachable!(), + _ => codegen_error!("singlepass Operator::LocalSet unreachable"), }, loc, self.locals[local_index], - ); + ) } else { self.machine - .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]); + .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]) } } else { self.machine - .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]); - } + .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]) + }?; } Operator::LocalTee { local_index } => { let local_index = local_index as usize; @@ -1302,19 +1355,19 @@ impl<'a, M: Machine> FuncGen<'a, M> { match self.local_types[local_index] { WpType::F32 => Size::S32, WpType::F64 => Size::S64, - _ => unreachable!(), + _ => codegen_error!("singlepass Operator::LocalTee unreachable"), }, loc, self.locals[local_index], - ); + ) } else { self.machine - .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]); + .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]) } } else { self.machine - .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]); - } + .emit_relaxed_mov(Size::S64, loc, self.locals[local_index]) + }?; } Operator::I32Const { value } => { self.value_stack.push(Location::Imm32(value as u32)); @@ -1323,168 +1376,168 @@ impl<'a, M: Machine> FuncGen<'a, M> { .push(WasmAbstractValue::Const(value as u32 as u64)); } Operator::I32Add => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_add32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_add32(loc_a, loc_b, ret)?; } Operator::I32Sub => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_sub32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_sub32(loc_a, loc_b, ret)?; } Operator::I32Mul => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_mul32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_mul32(loc_a, loc_b, ret)?; } Operator::I32DivU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; let offset = self.machine.emit_binop_udiv32( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I32DivS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; let offset = self.machine.emit_binop_sdiv32( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I32RemU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; let offset = self.machine.emit_binop_urem32( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I32RemS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; let offset = self.machine.emit_binop_srem32( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I32And => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_and32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_and32(loc_a, loc_b, ret)?; } Operator::I32Or => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_or32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_or32(loc_a, loc_b, ret)?; } Operator::I32Xor => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.emit_binop_xor32(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.emit_binop_xor32(loc_a, loc_b, ret)?; } Operator::I32Eq => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_eq(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_eq(loc_a, loc_b, ret)?; } Operator::I32Ne => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_ne(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_ne(loc_a, loc_b, ret)?; } Operator::I32Eqz => { - let loc_a = self.pop_value_released(); + let loc_a = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; - self.machine.i32_cmp_eq(loc_a, Location::Imm32(0), ret); + )?[0]; + self.machine.i32_cmp_eq(loc_a, Location::Imm32(0), ret)?; self.value_stack.push(ret); } Operator::I32Clz => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i32_clz(loc, ret); + self.machine.i32_clz(loc, ret)?; } Operator::I32Ctz => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i32_ctz(loc, ret); + self.machine.i32_ctz(loc, ret)?; } Operator::I32Popcnt => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i32_popcnt(loc, ret); + self.machine.i32_popcnt(loc, ret)?; } Operator::I32Shl => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_shl(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_shl(loc_a, loc_b, ret)?; } Operator::I32ShrU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_shr(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_shr(loc_a, loc_b, ret)?; } Operator::I32ShrS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_sar(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_sar(loc_a, loc_b, ret)?; } Operator::I32Rotl => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_rol(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_rol(loc_a, loc_b, ret)?; } Operator::I32Rotr => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_ror(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_ror(loc_a, loc_b, ret)?; } Operator::I32LtU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_lt_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_lt_u(loc_a, loc_b, ret)?; } Operator::I32LeU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_le_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_le_u(loc_a, loc_b, ret)?; } Operator::I32GtU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_gt_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_gt_u(loc_a, loc_b, ret)?; } Operator::I32GeU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_ge_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_ge_u(loc_a, loc_b, ret)?; } Operator::I32LtS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_lt_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_lt_s(loc_a, loc_b, ret)?; } Operator::I32LeS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_le_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_le_s(loc_a, loc_b, ret)?; } Operator::I32GtS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_gt_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_gt_s(loc_a, loc_b, ret)?; } Operator::I32GeS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.i32_cmp_ge_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.i32_cmp_ge_s(loc_a, loc_b, ret)?; } Operator::I64Const { value } => { let value = value as u64; @@ -1492,177 +1545,177 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.state.wasm_stack.push(WasmAbstractValue::Const(value)); } Operator::I64Add => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_add64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_add64(loc_a, loc_b, ret)?; } Operator::I64Sub => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_sub64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_sub64(loc_a, loc_b, ret)?; } Operator::I64Mul => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_mul64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_mul64(loc_a, loc_b, ret)?; } Operator::I64DivU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; let offset = self.machine.emit_binop_udiv64( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I64DivS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; let offset = self.machine.emit_binop_sdiv64( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I64RemU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; let offset = self.machine.emit_binop_urem64( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I64RemS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; let offset = self.machine.emit_binop_srem64( loc_a, loc_b, ret, self.special_labels.integer_division_by_zero, self.special_labels.integer_overflow, - ); + )?; self.mark_offset_trappable(offset); } Operator::I64And => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_and64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_and64(loc_a, loc_b, ret)?; } Operator::I64Or => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_or64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_or64(loc_a, loc_b, ret)?; } Operator::I64Xor => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.emit_binop_xor64(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.emit_binop_xor64(loc_a, loc_b, ret)?; } Operator::I64Eq => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_eq(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_eq(loc_a, loc_b, ret)?; } Operator::I64Ne => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_ne(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_ne(loc_a, loc_b, ret)?; } Operator::I64Eqz => { - let loc_a = self.pop_value_released(); + let loc_a = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; - self.machine.i64_cmp_eq(loc_a, Location::Imm64(0), ret); + )?[0]; + self.machine.i64_cmp_eq(loc_a, Location::Imm64(0), ret)?; self.value_stack.push(ret); } Operator::I64Clz => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i64_clz(loc, ret); + self.machine.i64_clz(loc, ret)?; } Operator::I64Ctz => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i64_ctz(loc, ret); + self.machine.i64_ctz(loc, ret)?; } Operator::I64Popcnt => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.i64_popcnt(loc, ret); + self.machine.i64_popcnt(loc, ret)?; } Operator::I64Shl => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_shl(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_shl(loc_a, loc_b, ret)?; } Operator::I64ShrU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_shr(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_shr(loc_a, loc_b, ret)?; } Operator::I64ShrS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_sar(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_sar(loc_a, loc_b, ret)?; } Operator::I64Rotl => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_rol(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_rol(loc_a, loc_b, ret)?; } Operator::I64Rotr => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_ror(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_ror(loc_a, loc_b, ret)?; } Operator::I64LtU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_lt_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_lt_u(loc_a, loc_b, ret)?; } Operator::I64LeU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_le_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_le_u(loc_a, loc_b, ret)?; } Operator::I64GtU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_gt_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_gt_u(loc_a, loc_b, ret)?; } Operator::I64GeU => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_ge_u(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_ge_u(loc_a, loc_b, ret)?; } Operator::I64LtS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_lt_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_lt_s(loc_a, loc_b, ret)?; } Operator::I64LeS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_le_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_le_s(loc_a, loc_b, ret)?; } Operator::I64GtS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_gt_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_gt_s(loc_a, loc_b, ret)?; } Operator::I64GeS => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64); - self.machine.i64_cmp_ge_s(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I64)?; + self.machine.i64_cmp_ge_s(loc_a, loc_b, ret)?; } Operator::I64ExtendI32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.emit_relaxed_mov(Size::S32, loc, ret); + self.machine.emit_relaxed_mov(Size::S32, loc, ret)?; // A 32-bit memory write does not automatically clear the upper 32 bits of a 64-bit word. // So, we need to explicitly write zero to the upper half here. @@ -1671,82 +1724,82 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S32, Location::Imm32(0), Location::Memory(base, off + 4), - ); + )?; } } Operator::I64ExtendI32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S32, loc, Size::S64, ret); + .emit_relaxed_sign_extension(Size::S32, loc, Size::S64, ret)?; } Operator::I32Extend8S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S8, loc, Size::S32, ret); + .emit_relaxed_sign_extension(Size::S8, loc, Size::S32, ret)?; } Operator::I32Extend16S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S16, loc, Size::S32, ret); + .emit_relaxed_sign_extension(Size::S16, loc, Size::S32, ret)?; } Operator::I64Extend8S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S8, loc, Size::S64, ret); + .emit_relaxed_sign_extension(Size::S8, loc, Size::S64, ret)?; } Operator::I64Extend16S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S16, loc, Size::S64, ret); + .emit_relaxed_sign_extension(Size::S16, loc, Size::S64, ret)?; } Operator::I64Extend32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine - .emit_relaxed_sign_extension(Size::S32, loc, Size::S64, ret); + .emit_relaxed_sign_extension(Size::S32, loc, Size::S64, ret)?; } Operator::I32WrapI64 => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.emit_relaxed_mov(Size::S32, loc, ret); + self.machine.emit_relaxed_mov(Size::S32, loc, ret)?; } Operator::F32Const { value } => { @@ -1761,141 +1814,141 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f32_add(loc_a, loc_b, ret); + self.machine.f32_add(loc_a, loc_b, ret)?; } Operator::F32Sub => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f32_sub(loc_a, loc_b, ret); + self.machine.f32_sub(loc_a, loc_b, ret)?; } Operator::F32Mul => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f32_mul(loc_a, loc_b, ret); + self.machine.f32_mul(loc_a, loc_b, ret)?; } Operator::F32Div => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f32_div(loc_a, loc_b, ret); + self.machine.f32_div(loc_a, loc_b, ret)?; } Operator::F32Max => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); - self.machine.f32_max(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; + self.machine.f32_max(loc_a, loc_b, ret)?; } Operator::F32Min => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); - self.machine.f32_min(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; + self.machine.f32_min(loc_a, loc_b, ret)?; } Operator::F32Eq => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_eq(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_eq(loc_a, loc_b, ret)?; } Operator::F32Ne => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_ne(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_ne(loc_a, loc_b, ret)?; } Operator::F32Lt => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_lt(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_lt(loc_a, loc_b, ret)?; } Operator::F32Le => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_le(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_le(loc_a, loc_b, ret)?; } Operator::F32Gt => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_gt(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_gt(loc_a, loc_b, ret)?; } Operator::F32Ge => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f32_cmp_ge(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f32_cmp_ge(loc_a, loc_b, ret)?; } Operator::F32Nearest => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_nearest(loc, ret); + self.machine.f32_nearest(loc, ret)?; } Operator::F32Floor => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_floor(loc, ret); + self.machine.f32_floor(loc, ret)?; } Operator::F32Ceil => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_ceil(loc, ret); + self.machine.f32_ceil(loc, ret)?; } Operator::F32Trunc => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_trunc(loc, ret); + self.machine.f32_trunc(loc, ret)?; } Operator::F32Sqrt => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_sqrt(loc, ret); + self.machine.f32_sqrt(loc, ret)?; } Operator::F32Copysign => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F32); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F32)?; let (fp_src1, fp_src2) = self.fp_stack.pop2()?; self.fp_stack @@ -1911,23 +1964,23 @@ impl<'a, M: Machine> FuncGen<'a, M> { match fp.canonicalization { Some(_) => { self.machine - .canonicalize_nan(Size::S32, *loc, Location::GPR(*tmp)); + .canonicalize_nan(Size::S32, *loc, Location::GPR(*tmp)) } None => { self.machine - .move_location(Size::S32, *loc, Location::GPR(*tmp)); + .move_location(Size::S32, *loc, Location::GPR(*tmp)) } - } + }?; } } else { self.machine - .move_location(Size::S32, loc_a, Location::GPR(tmp1)); + .move_location(Size::S32, loc_a, Location::GPR(tmp1))?; self.machine - .move_location(Size::S32, loc_b, Location::GPR(tmp2)); + .move_location(Size::S32, loc_b, Location::GPR(tmp2))?; } - self.machine.emit_i32_copysign(tmp1, tmp2); + self.machine.emit_i32_copysign(tmp1, tmp2)?; self.machine - .move_location(Size::S32, Location::GPR(tmp1), ret); + .move_location(Size::S32, Location::GPR(tmp1), ret)?; self.machine.release_gpr(tmp2); self.machine.release_gpr(tmp1); } @@ -1935,27 +1988,27 @@ impl<'a, M: Machine> FuncGen<'a, M> { Operator::F32Abs => { // Preserve canonicalization state. - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_abs(loc, ret); + self.machine.f32_abs(loc, ret)?; } Operator::F32Neg => { // Preserve canonicalization state. - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f32_neg(loc, ret); + self.machine.f32_neg(loc, ret)?; } Operator::F64Const { value } => { @@ -1970,141 +2023,141 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f64_add(loc_a, loc_b, ret); + self.machine.f64_add(loc_a, loc_b, ret)?; } Operator::F64Sub => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f64_sub(loc_a, loc_b, ret); + self.machine.f64_sub(loc_a, loc_b, ret)?; } Operator::F64Mul => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f64_mul(loc_a, loc_b, ret); + self.machine.f64_mul(loc_a, loc_b, ret)?; } Operator::F64Div => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; - self.machine.f64_div(loc_a, loc_b, ret); + self.machine.f64_div(loc_a, loc_b, ret)?; } Operator::F64Max => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); - self.machine.f64_max(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; + self.machine.f64_max(loc_a, loc_b, ret)?; } Operator::F64Min => { self.fp_stack.pop2()?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 2)); - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); - self.machine.f64_min(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; + self.machine.f64_min(loc_a, loc_b, ret)?; } Operator::F64Eq => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_eq(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_eq(loc_a, loc_b, ret)?; } Operator::F64Ne => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_ne(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_ne(loc_a, loc_b, ret)?; } Operator::F64Lt => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_lt(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_lt(loc_a, loc_b, ret)?; } Operator::F64Le => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_le(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_le(loc_a, loc_b, ret)?; } Operator::F64Gt => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_gt(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_gt(loc_a, loc_b, ret)?; } Operator::F64Ge => { self.fp_stack.pop2()?; - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32); - self.machine.f64_cmp_ge(loc_a, loc_b, ret); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::I32)?; + self.machine.f64_cmp_ge(loc_a, loc_b, ret)?; } Operator::F64Nearest => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_nearest(loc, ret); + self.machine.f64_nearest(loc, ret)?; } Operator::F64Floor => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_floor(loc, ret); + self.machine.f64_floor(loc, ret)?; } Operator::F64Ceil => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_ceil(loc, ret); + self.machine.f64_ceil(loc, ret)?; } Operator::F64Trunc => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_trunc(loc, ret); + self.machine.f64_trunc(loc, ret)?; } Operator::F64Sqrt => { self.fp_stack.pop1()?; self.fp_stack .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_sqrt(loc, ret); + self.machine.f64_sqrt(loc, ret)?; } Operator::F64Copysign => { - let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64); + let I2O1 { loc_a, loc_b, ret } = self.i2o1_prepare(WpType::F64)?; let (fp_src1, fp_src2) = self.fp_stack.pop2()?; self.fp_stack @@ -2120,23 +2173,23 @@ impl<'a, M: Machine> FuncGen<'a, M> { match fp.canonicalization { Some(_) => { self.machine - .canonicalize_nan(Size::S64, *loc, Location::GPR(*tmp)); + .canonicalize_nan(Size::S64, *loc, Location::GPR(*tmp)) } None => { self.machine - .move_location(Size::S64, *loc, Location::GPR(*tmp)); + .move_location(Size::S64, *loc, Location::GPR(*tmp)) } - } + }?; } } else { self.machine - .move_location(Size::S64, loc_a, Location::GPR(tmp1)); + .move_location(Size::S64, loc_a, Location::GPR(tmp1))?; self.machine - .move_location(Size::S64, loc_b, Location::GPR(tmp2)); + .move_location(Size::S64, loc_b, Location::GPR(tmp2))?; } - self.machine.emit_i64_copysign(tmp1, tmp2); + self.machine.emit_i64_copysign(tmp1, tmp2)?; self.machine - .move_location(Size::S64, Location::GPR(tmp1), ret); + .move_location(Size::S64, Location::GPR(tmp1), ret)?; self.machine.release_gpr(tmp2); self.machine.release_gpr(tmp1); @@ -2145,58 +2198,58 @@ impl<'a, M: Machine> FuncGen<'a, M> { Operator::F64Abs => { // Preserve canonicalization state. - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_abs(loc, ret); + self.machine.f64_abs(loc, ret)?; } Operator::F64Neg => { // Preserve canonicalization state. - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.f64_neg(loc, ret); + self.machine.f64_neg(loc, ret)?; } Operator::F64PromoteF32 => { let fp = self.fp_stack.pop1()?; - self.fp_stack.push(fp.promote(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + self.fp_stack.push(fp.promote(self.value_stack.len() - 1)?); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.convert_f64_f32(loc, ret); + self.machine.convert_f64_f32(loc, ret)?; } Operator::F32DemoteF64 => { let fp = self.fp_stack.pop1()?; - self.fp_stack.push(fp.demote(self.value_stack.len() - 1)); - let loc = self.pop_value_released(); + self.fp_stack.push(fp.demote(self.value_stack.len() - 1)?); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); - self.machine.convert_f32_f64(loc, ret); + self.machine.convert_f32_f64(loc, ret)?; } Operator::I32ReinterpretF32 => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); let fp = self.fp_stack.pop1()?; @@ -2205,33 +2258,33 @@ impl<'a, M: Machine> FuncGen<'a, M> { || fp.canonicalization.is_none() { if loc != ret { - self.machine.emit_relaxed_mov(Size::S32, loc, ret); + self.machine.emit_relaxed_mov(Size::S32, loc, ret)?; } } else { - self.machine.canonicalize_nan(Size::S32, loc, ret); + self.machine.canonicalize_nan(Size::S32, loc, ret)?; } } Operator::F32ReinterpretI32 => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); if loc != ret { - self.machine.emit_relaxed_mov(Size::S32, loc, ret); + self.machine.emit_relaxed_mov(Size::S32, loc, ret)?; } } Operator::I64ReinterpretF64 => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); let fp = self.fp_stack.pop1()?; @@ -2240,313 +2293,313 @@ impl<'a, M: Machine> FuncGen<'a, M> { || fp.canonicalization.is_none() { if loc != ret { - self.machine.emit_relaxed_mov(Size::S64, loc, ret); + self.machine.emit_relaxed_mov(Size::S64, loc, ret)?; } } else { - self.machine.canonicalize_nan(Size::S64, loc, ret); + self.machine.canonicalize_nan(Size::S64, loc, ret)?; } } Operator::F64ReinterpretI64 => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); if loc != ret { - self.machine.emit_relaxed_mov(Size::S64, loc, ret); + self.machine.emit_relaxed_mov(Size::S64, loc, ret)?; } } Operator::I32TruncF32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f32(loc, ret, false, false); + self.machine.convert_i32_f32(loc, ret, false, false)?; } Operator::I32TruncSatF32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f32(loc, ret, false, true); + self.machine.convert_i32_f32(loc, ret, false, true)?; } Operator::I32TruncF32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f32(loc, ret, true, false); + self.machine.convert_i32_f32(loc, ret, true, false)?; } Operator::I32TruncSatF32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f32(loc, ret, true, true); + self.machine.convert_i32_f32(loc, ret, true, true)?; } Operator::I64TruncF32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f32(loc, ret, true, false); + self.machine.convert_i64_f32(loc, ret, true, false)?; } Operator::I64TruncSatF32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f32(loc, ret, true, true); + self.machine.convert_i64_f32(loc, ret, true, true)?; } Operator::I64TruncF32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f32(loc, ret, false, false); + self.machine.convert_i64_f32(loc, ret, false, false)?; } Operator::I64TruncSatF32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f32(loc, ret, false, true); + self.machine.convert_i64_f32(loc, ret, false, true)?; } Operator::I32TruncF64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f64(loc, ret, false, false); + self.machine.convert_i32_f64(loc, ret, false, false)?; } Operator::I32TruncSatF64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f64(loc, ret, false, true); + self.machine.convert_i32_f64(loc, ret, false, true)?; } Operator::I32TruncF64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f64(loc, ret, true, false); + self.machine.convert_i32_f64(loc, ret, true, false)?; } Operator::I32TruncSatF64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i32_f64(loc, ret, true, true); + self.machine.convert_i32_f64(loc, ret, true, true)?; } Operator::I64TruncF64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f64(loc, ret, true, false); + self.machine.convert_i64_f64(loc, ret, true, false)?; } Operator::I64TruncSatF64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f64(loc, ret, true, true); + self.machine.convert_i64_f64(loc, ret, true, true)?; } Operator::I64TruncF64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f64(loc, ret, false, false); + self.machine.convert_i64_f64(loc, ret, false, false)?; } Operator::I64TruncSatF64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack.pop1()?; - self.machine.convert_i64_f64(loc, ret, false, true); + self.machine.convert_i64_f64(loc, ret, false, true)?; } Operator::F32ConvertI32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f32 never results in NaN. - self.machine.convert_f32_i32(loc, true, ret); + self.machine.convert_f32_i32(loc, true, ret)?; } Operator::F32ConvertI32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f32 never results in NaN. - self.machine.convert_f32_i32(loc, false, ret); + self.machine.convert_f32_i32(loc, false, ret)?; } Operator::F32ConvertI64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f32 never results in NaN. - self.machine.convert_f32_i64(loc, true, ret); + self.machine.convert_f32_i64(loc, true, ret)?; } Operator::F32ConvertI64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f32 never results in NaN. - self.machine.convert_f32_i64(loc, false, ret); + self.machine.convert_f32_i64(loc, false, ret)?; } Operator::F64ConvertI32S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f64 never results in NaN. - self.machine.convert_f64_i32(loc, true, ret); + self.machine.convert_f64_i32(loc, true, ret)?; } Operator::F64ConvertI32U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f64 never results in NaN. - self.machine.convert_f64_i32(loc, false, ret); + self.machine.convert_f64_i32(loc, false, ret)?; } Operator::F64ConvertI64S => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f64 never results in NaN. - self.machine.convert_f64_i64(loc, true, ret); + self.machine.convert_f64_i64(loc, true, ret)?; } Operator::F64ConvertI64U => { - let loc = self.pop_value_released(); + let loc = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f64 never results in NaN. - self.machine.convert_f64_i64(loc, false, ret); + self.machine.convert_f64_i64(loc, false, ret)?; } Operator::Call { function_index } => { @@ -2567,9 +2620,9 @@ impl<'a, M: Machine> FuncGen<'a, M> { .value_stack .drain(self.value_stack.len() - param_types.len()..) .collect(); - self.release_locations_only_regs(¶ms); + self.release_locations_only_regs(¶ms)?; - self.release_locations_only_osr_state(params.len()); + self.release_locations_only_osr_state(params.len())?; // Pop arguments off the FP stack and canonicalize them if needed. // @@ -2584,7 +2637,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { { let size = fp.canonicalization.unwrap().to_size(); self.machine - .canonicalize_nan(size, params[index], params[index]); + .canonicalize_nan(size, params[index], params[index])?; } self.fp_stack.pop().unwrap(); } else { @@ -2609,15 +2662,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { .mark_instruction_with_trap_code(TrapCode::StackOverflow); let mut relocations = this .machine - .emit_call_with_reloc(calling_convention, reloc_target); + .emit_call_with_reloc(calling_convention, reloc_target)?; this.machine.mark_instruction_address_end(offset); this.relocations.append(&mut relocations); + Ok(()) }, params.iter().copied(), param_types.iter().copied(), )?; - self.release_locations_only_stack(¶ms); + self.release_locations_only_stack(¶ms)?; if !return_types.is_empty() { let ret = self.acquire_locations( @@ -2626,14 +2680,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { MachineValue::WasmStack(self.value_stack.len()), )], false, - )[0]; + )?[0]; self.value_stack.push(ret); if return_types[0].is_float() { self.machine.move_location( Size::S64, Location::SIMD(self.machine.get_simd_for_ret()), ret, - ); + )?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); } else { @@ -2641,7 +2695,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::GPR(self.machine.get_gpr_for_ret()), ret, - ); + )?; } } } @@ -2656,13 +2710,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { let return_types: SmallVec<[WpType; 1]> = sig.results().iter().cloned().map(type_to_wp_type).collect(); - let func_index = self.pop_value_released(); + let func_index = self.pop_value_released()?; let params: SmallVec<[_; 8]> = self .value_stack .drain(self.value_stack.len() - param_types.len()..) .collect(); - self.release_locations_only_regs(¶ms); + self.release_locations_only_regs(¶ms)?; // Pop arguments off the FP stack and canonicalize them if needed. // @@ -2677,7 +2731,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { { let size = fp.canonicalization.unwrap().to_size(); self.machine - .canonicalize_nan(size, params[index], params[index]); + .canonicalize_nan(size, params[index], params[index])?; } self.fp_stack.pop().unwrap(); } else { @@ -2699,12 +2753,12 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), vmctx_offset_base as i32), Location::GPR(table_base), - ); + )?; self.machine.move_location( Size::S32, Location::Memory(self.machine.get_vmctx_reg(), vmctx_offset_len as i32), Location::GPR(table_count), - ); + )?; } else { // Do an indirection. let import_offset = self.vmoffsets.vmctx_vmtable_import(table_index); @@ -2712,7 +2766,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::Memory(self.machine.get_vmctx_reg(), import_offset as i32), Location::GPR(table_base), - ); + )?; // Load len. self.machine.move_location( @@ -2722,48 +2776,48 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.vmoffsets.vmtable_definition_current_elements() as _, ), Location::GPR(table_count), - ); + )?; // Load base. self.machine.move_location( Size::S64, Location::Memory(table_base, self.vmoffsets.vmtable_definition_base() as _), Location::GPR(table_base), - ); + )?; } self.machine - .location_cmp(Size::S32, func_index, Location::GPR(table_count)); + .location_cmp(Size::S32, func_index, Location::GPR(table_count))?; self.machine - .jmp_on_belowequal(self.special_labels.table_access_oob); + .jmp_on_belowequal(self.special_labels.table_access_oob)?; self.machine - .move_location(Size::S32, func_index, Location::GPR(table_count)); + .move_location(Size::S32, func_index, Location::GPR(table_count))?; self.machine.emit_imul_imm32( Size::S64, self.vmoffsets.size_of_vm_funcref() as u32, table_count, - ); + )?; self.machine.location_add( Size::S64, Location::GPR(table_base), Location::GPR(table_count), false, - ); + )?; // deref the table to get a VMFuncRef self.machine.move_location( Size::S64, Location::Memory(table_count, self.vmoffsets.vm_funcref_anyfunc_ptr() as i32), Location::GPR(table_count), - ); + )?; // Trap if the FuncRef is null self.machine.location_cmp( Size::S64, Location::Imm32(0), Location::GPR(table_count), - ); + )?; self.machine - .jmp_on_equal(self.special_labels.indirect_call_null); + .jmp_on_equal(self.special_labels.indirect_call_null)?; self.machine.move_location( Size::S64, Location::Memory( @@ -2771,7 +2825,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.vmoffsets.vmctx_vmshared_signature_id(index) as i32, ), Location::GPR(sigidx), - ); + )?; // Trap if signature mismatches. self.machine.location_cmp( @@ -2781,9 +2835,9 @@ impl<'a, M: Machine> FuncGen<'a, M> { table_count, (self.vmoffsets.vmcaller_checked_anyfunc_type_index() as usize) as i32, ), - ); + )?; self.machine - .jmp_on_different(self.special_labels.bad_signature); + .jmp_on_different(self.special_labels.bad_signature)?; self.machine.release_gpr(sigidx); self.machine.release_gpr(table_count); @@ -2795,10 +2849,10 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::GPR(table_count), Location::GPR(gpr_for_call), - ); + )?; } - self.release_locations_only_osr_state(params.len()); + self.release_locations_only_osr_state(params.len())?; let vmcaller_checked_anyfunc_func_ptr = self.vmoffsets.vmcaller_checked_anyfunc_func_ptr() as usize; @@ -2813,7 +2867,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { .arch_emit_indirect_call_with_trampoline(Location::Memory( gpr_for_call, vmcaller_checked_anyfunc_func_ptr as i32, - )); + )) } else { let offset = this .machine @@ -2828,20 +2882,21 @@ impl<'a, M: Machine> FuncGen<'a, M> { ), this.machine .get_simple_param_location(0, calling_convention), - ); + )?; this.machine.emit_call_location(Location::Memory( gpr_for_call, vmcaller_checked_anyfunc_func_ptr as i32, - )); + ))?; this.machine.mark_instruction_address_end(offset); + Ok(()) } }, params.iter().copied(), param_types.iter().copied(), )?; - self.release_locations_only_stack(¶ms); + self.release_locations_only_stack(¶ms)?; if !return_types.is_empty() { let ret = self.acquire_locations( @@ -2850,14 +2905,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { MachineValue::WasmStack(self.value_stack.len()), )], false, - )[0]; + )?[0]; self.value_stack.push(ret); if return_types[0].is_float() { self.machine.move_location( Size::S64, Location::SIMD(self.machine.get_simd_for_ret()), ret, - ); + )?; self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); } else { @@ -2865,7 +2920,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { Size::S64, Location::GPR(self.machine.get_gpr_for_ret()), ret, - ); + )?; } } } @@ -2873,7 +2928,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { let label_end = self.machine.get_label(); let label_else = self.machine.get_label(); - let cond = self.pop_value_released(); + let cond = self.pop_value_released()?; let frame = ControlFrame { label: label_end, @@ -2895,8 +2950,8 @@ impl<'a, M: Machine> FuncGen<'a, M> { }; self.control_stack.push(frame); self.machine - .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond); - self.machine.jmp_on_equal(label_else); + .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond)?; + self.machine.jmp_on_equal(label_else)?; } Operator::Else => { let frame = self.control_stack.last_mut().unwrap(); @@ -2913,21 +2968,21 @@ impl<'a, M: Machine> FuncGen<'a, M> { false }; self.machine - .emit_function_return_value(first_return, canonicalize, loc); + .emit_function_return_value(first_return, canonicalize, loc)?; } let frame = &self.control_stack.last_mut().unwrap(); let stack_depth = frame.value_stack_depth; let fp_depth = frame.fp_stack_depth; - self.release_locations_value(stack_depth); + self.release_locations_value(stack_depth)?; self.value_stack.truncate(stack_depth); self.fp_stack.truncate(fp_depth); let mut frame = &mut self.control_stack.last_mut().unwrap(); match frame.if_else { IfElseState::If(label) => { - self.machine.jmp_unconditionnal(frame.label); - self.machine.emit_label(label); + self.machine.jmp_unconditionnal(frame.label)?; + self.machine.emit_label(label)?; frame.if_else = IfElseState::Else; } _ => { @@ -2940,9 +2995,9 @@ impl<'a, M: Machine> FuncGen<'a, M> { // `TypedSelect` must be used for extern refs so ref counting should // be done with TypedSelect. But otherwise they're the same. Operator::TypedSelect { .. } | Operator::Select => { - let cond = self.pop_value_released(); - let v_b = self.pop_value_released(); - let v_a = self.pop_value_released(); + let cond = self.pop_value_released()?; + let v_b = self.pop_value_released()?; + let v_a = self.pop_value_released()?; let cncl: Option<(Option, Option)> = if self.fp_stack.len() >= 2 && self.fp_stack[self.fp_stack.len() - 2].depth == self.value_stack.len() @@ -2958,44 +3013,44 @@ impl<'a, M: Machine> FuncGen<'a, M> { let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); let end_label = self.machine.get_label(); let zero_label = self.machine.get_label(); self.machine - .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond); - self.machine.jmp_on_equal(zero_label); + .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond)?; + self.machine.jmp_on_equal(zero_label)?; match cncl { Some((Some(fp), _)) if self.machine.arch_supports_canonicalize_nan() && self.config.enable_nan_canonicalization => { - self.machine.canonicalize_nan(fp.to_size(), v_a, ret); + self.machine.canonicalize_nan(fp.to_size(), v_a, ret)?; } _ => { if v_a != ret { - self.machine.emit_relaxed_mov(Size::S64, v_a, ret); + self.machine.emit_relaxed_mov(Size::S64, v_a, ret)?; } } } - self.machine.jmp_unconditionnal(end_label); - self.machine.emit_label(zero_label); + self.machine.jmp_unconditionnal(end_label)?; + self.machine.emit_label(zero_label)?; match cncl { Some((_, Some(fp))) if self.machine.arch_supports_canonicalize_nan() && self.config.enable_nan_canonicalization => { - self.machine.canonicalize_nan(fp.to_size(), v_b, ret); + self.machine.canonicalize_nan(fp.to_size(), v_b, ret)?; } _ => { if v_b != ret { - self.machine.emit_relaxed_mov(Size::S64, v_b, ret); + self.machine.emit_relaxed_mov(Size::S64, v_b, ret)?; } } } - self.machine.emit_label(end_label); + self.machine.emit_label(end_label)?; } Operator::Block { ty } => { let frame = ControlFrame { @@ -3020,7 +3075,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.control_stack.push(frame); } Operator::Loop { ty } => { - self.machine.align_for_loop(); + self.machine.align_for_loop()?; let label = self.machine.get_label(); let state_diff_id = self.get_state_diff(); let _activate_offset = self.machine.assembler_get_offset().0; @@ -3044,7 +3099,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { state: self.state.clone(), state_diff_id, }); - self.machine.emit_label(label); + self.machine.emit_label(label)?; // TODO: Re-enable interrupt signal check without branching } @@ -3064,11 +3119,11 @@ impl<'a, M: Machine> FuncGen<'a, M> { ) as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, memory_index] iter::once(Location::Imm32(memory_index.index() as u32)), @@ -3077,19 +3132,19 @@ impl<'a, M: Machine> FuncGen<'a, M> { let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine.move_location( Size::S64, Location::GPR(self.machine.get_gpr_for_ret()), ret, - ); + )?; } Operator::MemoryInit { segment, mem } => { let len = self.value_stack.pop().unwrap(); let src = self.value_stack.pop().unwrap(); let dst = self.value_stack.pop().unwrap(); - self.release_locations_only_regs(&[len, src, dst]); + self.release_locations_only_regs(&[len, src, dst])?; self.machine.move_location( Size::S64, @@ -3100,15 +3155,15 @@ impl<'a, M: Machine> FuncGen<'a, M> { as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; // TODO: should this be 3? - self.release_locations_only_osr_state(1); + self.release_locations_only_osr_state(1)?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, memory_index, segment_index, dst, src, len] [ @@ -3130,7 +3185,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { .iter() .cloned(), )?; - self.release_locations_only_stack(&[dst, src, len]); + self.release_locations_only_stack(&[dst, src, len])?; } Operator::DataDrop { segment } => { self.machine.move_location( @@ -3142,12 +3197,12 @@ impl<'a, M: Machine> FuncGen<'a, M> { as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, segment_index] iter::once(Location::Imm32(segment)), @@ -3160,7 +3215,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { let len = self.value_stack.pop().unwrap(); let src_pos = self.value_stack.pop().unwrap(); let dst_pos = self.value_stack.pop().unwrap(); - self.release_locations_only_regs(&[len, src_pos, dst_pos]); + self.release_locations_only_regs(&[len, src_pos, dst_pos])?; let memory_index = MemoryIndex::new(src as usize); let (memory_copy_index, memory_index) = @@ -3183,15 +3238,15 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.vmoffsets.vmctx_builtin_function(memory_copy_index) as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; // TODO: should this be 3? - self.release_locations_only_osr_state(1); + self.release_locations_only_osr_state(1)?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, memory_index, dst, src, len] [ @@ -3206,13 +3261,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { .iter() .cloned(), )?; - self.release_locations_only_stack(&[dst_pos, src_pos, len]); + self.release_locations_only_stack(&[dst_pos, src_pos, len])?; } Operator::MemoryFill { mem } => { let len = self.value_stack.pop().unwrap(); let val = self.value_stack.pop().unwrap(); let dst = self.value_stack.pop().unwrap(); - self.release_locations_only_regs(&[len, val, dst]); + self.release_locations_only_regs(&[len, val, dst])?; let memory_index = MemoryIndex::new(mem as usize); let (memory_fill_index, memory_index) = @@ -3235,15 +3290,15 @@ impl<'a, M: Machine> FuncGen<'a, M> { self.vmoffsets.vmctx_builtin_function(memory_fill_index) as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; // TODO: should this be 3? - self.release_locations_only_osr_state(1); + self.release_locations_only_osr_state(1)?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, memory_index, dst, src, len] [Location::Imm32(memory_index.index() as u32), dst, val, len] @@ -3253,13 +3308,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { .iter() .cloned(), )?; - self.release_locations_only_stack(&[dst, val, len]); + self.release_locations_only_stack(&[dst, val, len])?; } Operator::MemoryGrow { mem, mem_byte: _ } => { let memory_index = MemoryIndex::new(mem as usize); let param_pages = self.value_stack.pop().unwrap(); - self.release_locations_only_regs(&[param_pages]); + self.release_locations_only_regs(&[param_pages])?; self.machine.move_location( Size::S64, @@ -3274,14 +3329,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { ) as i32, ), Location::GPR(self.machine.get_grp_for_call()), - ); + )?; - self.release_locations_only_osr_state(1); + self.release_locations_only_osr_state(1)?; self.emit_call_native( |this| { this.machine - .emit_call_register(this.machine.get_grp_for_call()); + .emit_call_register(this.machine.get_grp_for_call()) }, // [vmctx, val, memory_index] iter::once(param_pages) @@ -3289,25 +3344,25 @@ impl<'a, M: Machine> FuncGen<'a, M> { [WpType::I64, WpType::I64].iter().cloned(), )?; - self.release_locations_only_stack(&[param_pages]); + self.release_locations_only_stack(&[param_pages])?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.machine.move_location( Size::S64, Location::GPR(self.machine.get_gpr_for_ret()), ret, - ); + )?; } Operator::I32Load { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3319,16 +3374,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::F32Load { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); @@ -3342,16 +3397,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Load8U { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3363,16 +3418,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Load8S { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3384,16 +3439,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Load16U { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3405,16 +3460,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Load16S { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3426,13 +3481,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Store { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i32_save( @@ -3443,13 +3498,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::F32Store { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; let fp = self.fp_stack.pop1()?; let config_nan_canonicalization = self.config.enable_nan_canonicalization; self.op_memory( @@ -3463,13 +3518,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Store8 { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i32_save_8( @@ -3480,13 +3535,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I32Store16 { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i32_save_16( @@ -3497,16 +3552,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3518,16 +3573,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::F64Load { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); @@ -3541,16 +3596,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load8U { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3562,16 +3617,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load8S { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3583,16 +3638,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load16U { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3604,16 +3659,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load16S { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3625,16 +3680,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load32U { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3646,16 +3701,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Load32S { ref memarg } => { - let target = self.pop_value_released(); + let target = self.pop_value_released()?; let ret = self.acquire_locations( &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, - )[0]; + )?[0]; self.value_stack.push(ret); self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3667,13 +3722,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Store { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { @@ -3685,13 +3740,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::F64Store { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; let fp = self.fp_stack.pop1()?; let config_nan_canonicalization = self.config.enable_nan_canonicalization; self.op_memory( @@ -3705,13 +3760,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Store8 { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i64_save_8( @@ -3722,13 +3777,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Store16 { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i64_save_16( @@ -3739,13 +3794,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::I64Store32 { ref memarg } => { - let target_value = self.pop_value_released(); - let target_addr = self.pop_value_released(); + let target_value = self.pop_value_released()?; + let target_addr = self.pop_value_released()?; self.op_memory( |this, need_check, imported_memories, offset, heap_access_oob| { this.machine.i64_save_32( @@ -3756,14 +3811,14 @@ impl<'a, M: Machine> FuncGen<'a, M> { imported_memories, offset, heap_access_oob, - ); + ) }, - ); + )?; } Operator::Unreachable => { self.mark_trappable(); self.machine - .emit_illegal_op(TrapCode::UnreachableCodeReached); + .emit_illegal_op(TrapCode::UnreachableCodeReached)?; self.unreachable_depth = 1; } Operator::Return => { @@ -3785,13 +3840,13 @@ impl<'a, M: Machine> FuncGen<'a, M> { false }; self.machine - .emit_function_return_value(first_return, canonicalize, loc); + .emit_function_return_value(first_return, canonicalize, loc)?; } let frame = &self.control_stack[0]; let frame_depth = frame.value_stack_depth; let label = frame.label; - self.release_locations_keep_state(frame_depth); - self.machine.jmp_unconditionnal(label); + self.release_locations_keep_state(frame_depth)?; + self.machine.jmp_unconditionnal(label)?; self.unreachable_depth = 1; } Operator::Br { relative_depth } => { @@ -3814,23 +3869,23 @@ impl<'a, M: Machine> FuncGen<'a, M> { false }; self.machine - .emit_function_return_value(first_return, canonicalize, loc); + .emit_function_return_value(first_return, canonicalize, loc)?; } let stack_len = self.control_stack.len(); let frame = &mut self.control_stack[stack_len - 1 - (relative_depth as usize)]; let frame_depth = frame.value_stack_depth; let label = frame.label; - self.release_locations_keep_state(frame_depth); - self.machine.jmp_unconditionnal(label); + self.release_locations_keep_state(frame_depth)?; + self.machine.jmp_unconditionnal(label)?; self.unreachable_depth = 1; } Operator::BrIf { relative_depth } => { let after = self.machine.get_label(); - let cond = self.pop_value_released(); + let cond = self.pop_value_released()?; self.machine - .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond); - self.machine.jmp_on_equal(after); + .emit_relaxed_cmp(Size::S32, Location::Imm32(0), cond)?; + self.machine.jmp_on_equal(after)?; let frame = &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; @@ -3852,16 +3907,16 @@ impl<'a, M: Machine> FuncGen<'a, M> { false }; self.machine - .emit_function_return_value(first_return, canonicalize, loc); + .emit_function_return_value(first_return, canonicalize, loc)?; } let stack_len = self.control_stack.len(); let frame = &mut self.control_stack[stack_len - 1 - (relative_depth as usize)]; let stack_depth = frame.value_stack_depth; let label = frame.label; - self.release_locations_keep_state(stack_depth); - self.machine.jmp_unconditionnal(label); + self.release_locations_keep_state(stack_depth)?; + self.machine.jmp_unconditionnal(label)?; - self.machine.emit_label(after); + self.machine.emit_label(after)?; } Operator::BrTable { ref table } => { let targets = table @@ -3871,7 +3926,7 @@ impl<'a, M: Machine> FuncGen<'a, M> { message: format!("BrTable read_table: {:?}", e), })?; let default_target = table.default(); - let cond = self.pop_value_released(); + let cond = self.pop_value_released()?; let table_label = self.machine.get_label(); let mut table: Vec