1 //! This module contains the bulk of the interesting code performing the translation between
2 //! WebAssembly and Cranelift IR.
3 //!
4 //! The translation is done in one pass, opcode by opcode. Two main data structures are used during
5 //! code translations: the value stack and the control stack. The value stack mimics the execution
6 //! of the WebAssembly stack machine: each instruction result is pushed onto the stack and
7 //! instruction arguments are popped off the stack. Similarly, when encountering a control flow
8 //! block, it is pushed onto the control stack and popped off when encountering the corresponding
9 //! `End`.
10 //!
11 //! Another data structure, the translation state, records information concerning unreachable code
12 //! status and about if inserting a return at the end of the function is necessary.
13 //!
14 //! Some of the WebAssembly instructions need information about the environment for which they
15 //! are being translated:
16 //!
17 //! - the loads and stores need the memory base address;
18 //! - the `get_global` and `set_global` instructions depend on how the globals are implemented;
19 //! - `memory.size` and `memory.grow` are runtime functions;
20 //! - `call_indirect` has to translate the function index into the address of where this
21 //! is;
22 //!
23 //! That is why `translate_function_body` takes an object having the `WasmRuntime` trait as
24 //! argument.
25 //!
26 //! There is extra complexity associated with translation of 128-bit SIMD instructions.
27 //! Wasm only considers there to be a single 128-bit vector type. But CLIF's type system
28 //! distinguishes different lane configurations, so considers 8X16, 16X8, 32X4 and 64X2 to be
29 //! different types. The result is that, in wasm, it's perfectly OK to take the output of (eg)
30 //! an `add.16x8` and use that as an operand of a `sub.32x4`, without using any cast. But when
31 //! translated into CLIF, that will cause a verifier error due to the apparent type mismatch.
32 //!
33 //! This file works around that problem by liberally inserting `bitcast` instructions in many
34 //! places -- mostly, before the use of vector values, either as arguments to CLIF instructions
35 //! or as block actual parameters. These are no-op casts which nevertheless have different
36 //! input and output types, and are used (mostly) to "convert" 16X8, 32X4 and 64X2-typed vectors
37 //! to the "canonical" type, 8X16. Hence the functions `optionally_bitcast_vector`,
38 //! `bitcast_arguments`, `pop*_with_bitcast`, `canonicalise_then_jump`,
39 //! `canonicalise_then_br{z,nz}`, `is_non_canonical_v128` and `canonicalise_v128_values`.
40 //! Note that the `bitcast*` functions are occasionally used to convert to some type other than
41 //! 8X16, but the `canonicalise*` functions always convert to type 8X16.
42 //!
43 //! Be careful when adding support for new vector instructions. And when adding new jumps, even
44 //! if they are apparently don't have any connection to vectors. Never generate any kind of
45 //! (inter-block) jump directly. Instead use `canonicalise_then_jump` and
46 //! `canonicalise_then_br{z,nz}`.
47 //!
48 //! The use of bitcasts is ugly and inefficient, but currently unavoidable:
49 //!
50 //! * they make the logic in this file fragile: miss out a bitcast for any reason, and there is
51 //! the risk of the system failing in the verifier. At least for debug builds.
52 //!
53 //! * in the new backends, they potentially interfere with pattern matching on CLIF -- the
54 //! patterns need to take into account the presence of bitcast nodes.
55 //!
56 //! * in the new backends, they get translated into machine-level vector-register-copy
57 //! instructions, none of which are actually necessary. We then depend on the register
58 //! allocator to coalesce them all out.
59 //!
60 //! * they increase the total number of CLIF nodes that have to be processed, hence slowing down
61 //! the compilation pipeline. Also, the extra coalescing work generates a slowdown.
62 //!
63 //! A better solution which would avoid all four problems would be to remove the 8X16, 16X8,
64 //! 32X4 and 64X2 types from CLIF and instead have a single V128 type.
65 //!
66 //! For further background see also:
67 //! <https://github.com/bytecodealliance/wasmtime/issues/1147>
68 //! ("Too many raw_bitcasts in SIMD code")
69 //! <https://github.com/bytecodealliance/cranelift/pull/1251>
70 //! ("Add X128 type to represent WebAssembly's V128 type")
71 //! <https://github.com/bytecodealliance/cranelift/pull/1236>
72 //! ("Relax verification to allow I8X16 to act as a default vector type")
73
74 use super::{hash_map, HashMap};
75 use crate::environ::{FuncEnvironment, GlobalVariable, ReturnMode, WasmResult};
76 use crate::state::{ControlStackFrame, ElseData, FuncTranslationState};
77 use crate::translation_utils::{
78 block_with_params, blocktype_params_results, f32_translation, f64_translation,
79 };
80 use crate::translation_utils::{FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex};
81 use crate::wasm_unsupported;
82 use core::convert::TryInto;
83 use core::{i32, u32};
84 use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
85 use cranelift_codegen::ir::immediates::Offset32;
86 use cranelift_codegen::ir::types::*;
87 use cranelift_codegen::ir::{
88 self, AtomicRmwOp, ConstantData, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel,
89 };
90 use cranelift_codegen::packed_option::ReservedValue;
91 use cranelift_frontend::{FunctionBuilder, Variable};
92 use smallvec::SmallVec;
93 use std::cmp;
94 use std::convert::TryFrom;
95 use std::vec::Vec;
96 use wasmparser::{FuncValidator, MemoryImmediate, Operator, WasmModuleResources};
97
98 // Clippy warns about "align: _" but its important to document that the flags field is ignored
99 #[cfg_attr(
100 feature = "cargo-clippy",
101 allow(clippy::unneeded_field_pattern, clippy::cognitive_complexity)
102 )]
103 /// Translates wasm operators into Cranelift IR instructions. Returns `true` if it inserted
104 /// a return.
translate_operator<FE: FuncEnvironment + ?Sized>( validator: &mut FuncValidator<impl WasmModuleResources>, op: &Operator, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>105 pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
106 validator: &mut FuncValidator<impl WasmModuleResources>,
107 op: &Operator,
108 builder: &mut FunctionBuilder,
109 state: &mut FuncTranslationState,
110 environ: &mut FE,
111 ) -> WasmResult<()> {
112 if !state.reachable {
113 translate_unreachable_operator(validator, &op, builder, state, environ)?;
114 return Ok(());
115 }
116
117 // This big match treats all Wasm code operators.
118 match op {
119 /********************************** Locals ****************************************
120 * `get_local` and `set_local` are treated as non-SSA variables and will completely
121 * disappear in the Cranelift Code
122 ***********************************************************************************/
123 Operator::LocalGet { local_index } => {
124 let val = builder.use_var(Variable::with_u32(*local_index));
125 state.push1(val);
126 let label = ValueLabel::from_u32(*local_index);
127 builder.set_val_label(val, label);
128 }
129 Operator::LocalSet { local_index } => {
130 let mut val = state.pop1();
131
132 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
133 let ty = builder.func.dfg.value_type(val);
134 if ty.is_vector() {
135 val = optionally_bitcast_vector(val, I8X16, builder);
136 }
137
138 builder.def_var(Variable::with_u32(*local_index), val);
139 let label = ValueLabel::from_u32(*local_index);
140 builder.set_val_label(val, label);
141 }
142 Operator::LocalTee { local_index } => {
143 let mut val = state.peek1();
144
145 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
146 let ty = builder.func.dfg.value_type(val);
147 if ty.is_vector() {
148 val = optionally_bitcast_vector(val, I8X16, builder);
149 }
150
151 builder.def_var(Variable::with_u32(*local_index), val);
152 let label = ValueLabel::from_u32(*local_index);
153 builder.set_val_label(val, label);
154 }
155 /********************************** Globals ****************************************
156 * `get_global` and `set_global` are handled by the environment.
157 ***********************************************************************************/
158 Operator::GlobalGet { global_index } => {
159 let val = match state.get_global(builder.func, *global_index, environ)? {
160 GlobalVariable::Const(val) => val,
161 GlobalVariable::Memory { gv, offset, ty } => {
162 let addr = builder.ins().global_value(environ.pointer_type(), gv);
163 let flags = ir::MemFlags::trusted();
164 builder.ins().load(ty, flags, addr, offset)
165 }
166 GlobalVariable::Custom => environ.translate_custom_global_get(
167 builder.cursor(),
168 GlobalIndex::from_u32(*global_index),
169 )?,
170 };
171 state.push1(val);
172 }
173 Operator::GlobalSet { global_index } => {
174 match state.get_global(builder.func, *global_index, environ)? {
175 GlobalVariable::Const(_) => panic!("global #{} is a constant", *global_index),
176 GlobalVariable::Memory { gv, offset, ty } => {
177 let addr = builder.ins().global_value(environ.pointer_type(), gv);
178 let flags = ir::MemFlags::trusted();
179 let mut val = state.pop1();
180 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
181 if ty.is_vector() {
182 val = optionally_bitcast_vector(val, I8X16, builder);
183 }
184 debug_assert_eq!(ty, builder.func.dfg.value_type(val));
185 builder.ins().store(flags, val, addr, offset);
186 }
187 GlobalVariable::Custom => {
188 let val = state.pop1();
189 environ.translate_custom_global_set(
190 builder.cursor(),
191 GlobalIndex::from_u32(*global_index),
192 val,
193 )?;
194 }
195 }
196 }
197 /********************************* Stack misc ***************************************
198 * `drop`, `nop`, `unreachable` and `select`.
199 ***********************************************************************************/
200 Operator::Drop => {
201 state.pop1();
202 }
203 Operator::Select => {
204 let (mut arg1, mut arg2, cond) = state.pop3();
205 if builder.func.dfg.value_type(arg1).is_vector() {
206 arg1 = optionally_bitcast_vector(arg1, I8X16, builder);
207 }
208 if builder.func.dfg.value_type(arg2).is_vector() {
209 arg2 = optionally_bitcast_vector(arg2, I8X16, builder);
210 }
211 state.push1(builder.ins().select(cond, arg1, arg2));
212 }
213 Operator::TypedSelect { ty: _ } => {
214 // We ignore the explicit type parameter as it is only needed for
215 // validation, which we require to have been performed before
216 // translation.
217 let (mut arg1, mut arg2, cond) = state.pop3();
218 if builder.func.dfg.value_type(arg1).is_vector() {
219 arg1 = optionally_bitcast_vector(arg1, I8X16, builder);
220 }
221 if builder.func.dfg.value_type(arg2).is_vector() {
222 arg2 = optionally_bitcast_vector(arg2, I8X16, builder);
223 }
224 state.push1(builder.ins().select(cond, arg1, arg2));
225 }
226 Operator::Nop => {
227 // We do nothing
228 }
229 Operator::Unreachable => {
230 builder.ins().trap(ir::TrapCode::UnreachableCodeReached);
231 state.reachable = false;
232 }
233 /***************************** Control flow blocks **********************************
234 * When starting a control flow block, we create a new `Block` that will hold the code
235 * after the block, and we push a frame on the control stack. Depending on the type
236 * of block, we create a new `Block` for the body of the block with an associated
237 * jump instruction.
238 *
239 * The `End` instruction pops the last control frame from the control stack, seals
240 * the destination block (since `br` instructions targeting it only appear inside the
241 * block and have already been translated) and modify the value stack to use the
242 * possible `Block`'s arguments values.
243 ***********************************************************************************/
244 Operator::Block { ty } => {
245 let (params, results) = blocktype_params_results(validator, *ty)?;
246 let next = block_with_params(builder, results.clone(), environ)?;
247 state.push_block(next, params.len(), results.len());
248 }
249 Operator::Loop { ty } => {
250 let (params, results) = blocktype_params_results(validator, *ty)?;
251 let loop_body = block_with_params(builder, params.clone(), environ)?;
252 let next = block_with_params(builder, results.clone(), environ)?;
253 canonicalise_then_jump(builder, loop_body, state.peekn(params.len()));
254 state.push_loop(loop_body, next, params.len(), results.len());
255
256 // Pop the initial `Block` actuals and replace them with the `Block`'s
257 // params since control flow joins at the top of the loop.
258 state.popn(params.len());
259 state
260 .stack
261 .extend_from_slice(builder.block_params(loop_body));
262
263 builder.switch_to_block(loop_body);
264 environ.translate_loop_header(builder)?;
265 }
266 Operator::If { ty } => {
267 let val = state.pop1();
268
269 let (params, results) = blocktype_params_results(validator, *ty)?;
270 let (destination, else_data) = if params.clone().eq(results.clone()) {
271 // It is possible there is no `else` block, so we will only
272 // allocate a block for it if/when we find the `else`. For now,
273 // we if the condition isn't true, then we jump directly to the
274 // destination block following the whole `if...end`. If we do end
275 // up discovering an `else`, then we will allocate a block for it
276 // and go back and patch the jump.
277 let destination = block_with_params(builder, results.clone(), environ)?;
278 let branch_inst =
279 canonicalise_then_brz(builder, val, destination, state.peekn(params.len()));
280 (destination, ElseData::NoElse { branch_inst })
281 } else {
282 // The `if` type signature is not valid without an `else` block,
283 // so we eagerly allocate the `else` block here.
284 let destination = block_with_params(builder, results.clone(), environ)?;
285 let else_block = block_with_params(builder, params.clone(), environ)?;
286 canonicalise_then_brz(builder, val, else_block, state.peekn(params.len()));
287 builder.seal_block(else_block);
288 (destination, ElseData::WithElse { else_block })
289 };
290
291 let next_block = builder.create_block();
292 canonicalise_then_jump(builder, next_block, &[]);
293 builder.seal_block(next_block); // Only predecessor is the current block.
294 builder.switch_to_block(next_block);
295
296 // Here we append an argument to a Block targeted by an argumentless jump instruction
297 // But in fact there are two cases:
298 // - either the If does not have a Else clause, in that case ty = EmptyBlock
299 // and we add nothing;
300 // - either the If have an Else clause, in that case the destination of this jump
301 // instruction will be changed later when we translate the Else operator.
302 state.push_if(destination, else_data, params.len(), results.len(), *ty);
303 }
304 Operator::Else => {
305 let i = state.control_stack.len() - 1;
306 match state.control_stack[i] {
307 ControlStackFrame::If {
308 ref else_data,
309 head_is_reachable,
310 ref mut consequent_ends_reachable,
311 num_return_values,
312 blocktype,
313 destination,
314 ..
315 } => {
316 // We finished the consequent, so record its final
317 // reachability state.
318 debug_assert!(consequent_ends_reachable.is_none());
319 *consequent_ends_reachable = Some(state.reachable);
320
321 if head_is_reachable {
322 // We have a branch from the head of the `if` to the `else`.
323 state.reachable = true;
324
325 // Ensure we have a block for the `else` block (it may have
326 // already been pre-allocated, see `ElseData` for details).
327 let else_block = match *else_data {
328 ElseData::NoElse { branch_inst } => {
329 let (params, _results) =
330 blocktype_params_results(validator, blocktype)?;
331 debug_assert_eq!(params.len(), num_return_values);
332 let else_block =
333 block_with_params(builder, params.clone(), environ)?;
334 canonicalise_then_jump(
335 builder,
336 destination,
337 state.peekn(params.len()),
338 );
339 state.popn(params.len());
340
341 builder.change_jump_destination(branch_inst, else_block);
342 builder.seal_block(else_block);
343 else_block
344 }
345 ElseData::WithElse { else_block } => {
346 canonicalise_then_jump(
347 builder,
348 destination,
349 state.peekn(num_return_values),
350 );
351 state.popn(num_return_values);
352 else_block
353 }
354 };
355
356 // You might be expecting that we push the parameters for this
357 // `else` block here, something like this:
358 //
359 // state.pushn(&control_stack_frame.params);
360 //
361 // We don't do that because they are already on the top of the stack
362 // for us: we pushed the parameters twice when we saw the initial
363 // `if` so that we wouldn't have to save the parameters in the
364 // `ControlStackFrame` as another `Vec` allocation.
365
366 builder.switch_to_block(else_block);
367
368 // We don't bother updating the control frame's `ElseData`
369 // to `WithElse` because nothing else will read it.
370 }
371 }
372 _ => unreachable!(),
373 }
374 }
375 Operator::End => {
376 let frame = state.control_stack.pop().unwrap();
377 let next_block = frame.following_code();
378
379 if !builder.is_unreachable() || !builder.is_pristine() {
380 let return_count = frame.num_return_values();
381 let return_args = state.peekn_mut(return_count);
382 canonicalise_then_jump(builder, frame.following_code(), return_args);
383 // You might expect that if we just finished an `if` block that
384 // didn't have a corresponding `else` block, then we would clean
385 // up our duplicate set of parameters that we pushed earlier
386 // right here. However, we don't have to explicitly do that,
387 // since we truncate the stack back to the original height
388 // below.
389 }
390
391 builder.switch_to_block(next_block);
392 builder.seal_block(next_block);
393
394 // If it is a loop we also have to seal the body loop block
395 if let ControlStackFrame::Loop { header, .. } = frame {
396 builder.seal_block(header)
397 }
398
399 frame.truncate_value_stack_to_original_size(&mut state.stack);
400 state
401 .stack
402 .extend_from_slice(builder.block_params(next_block));
403 }
404 /**************************** Branch instructions *********************************
405 * The branch instructions all have as arguments a target nesting level, which
406 * corresponds to how many control stack frames do we have to pop to get the
407 * destination `Block`.
408 *
409 * Once the destination `Block` is found, we sometimes have to declare a certain depth
410 * of the stack unreachable, because some branch instructions are terminator.
411 *
412 * The `br_table` case is much more complicated because Cranelift's `br_table` instruction
413 * does not support jump arguments like all the other branch instructions. That is why, in
414 * the case where we would use jump arguments for every other branch instruction, we
415 * need to split the critical edges leaving the `br_tables` by creating one `Block` per
416 * table destination; the `br_table` will point to these newly created `Blocks` and these
417 * `Block`s contain only a jump instruction pointing to the final destination, this time with
418 * jump arguments.
419 *
420 * This system is also implemented in Cranelift's SSA construction algorithm, because
421 * `use_var` located in a destination `Block` of a `br_table` might trigger the addition
422 * of jump arguments in each predecessor branch instruction, one of which might be a
423 * `br_table`.
424 ***********************************************************************************/
425 Operator::Br { relative_depth } => {
426 let i = state.control_stack.len() - 1 - (*relative_depth as usize);
427 let (return_count, br_destination) = {
428 let frame = &mut state.control_stack[i];
429 // We signal that all the code that follows until the next End is unreachable
430 frame.set_branched_to_exit();
431 let return_count = if frame.is_loop() {
432 frame.num_param_values()
433 } else {
434 frame.num_return_values()
435 };
436 (return_count, frame.br_destination())
437 };
438 let destination_args = state.peekn_mut(return_count);
439 canonicalise_then_jump(builder, br_destination, destination_args);
440 state.popn(return_count);
441 state.reachable = false;
442 }
443 Operator::BrIf { relative_depth } => translate_br_if(*relative_depth, builder, state),
444 Operator::BrTable { table } => {
445 let mut depths = table.targets().collect::<Result<Vec<_>, _>>()?;
446 let default = depths.pop().unwrap().0;
447 let mut min_depth = default;
448 for (depth, _) in depths.iter() {
449 if *depth < min_depth {
450 min_depth = *depth;
451 }
452 }
453 let jump_args_count = {
454 let i = state.control_stack.len() - 1 - (min_depth as usize);
455 let min_depth_frame = &state.control_stack[i];
456 if min_depth_frame.is_loop() {
457 min_depth_frame.num_param_values()
458 } else {
459 min_depth_frame.num_return_values()
460 }
461 };
462 let val = state.pop1();
463 let mut data = JumpTableData::with_capacity(depths.len());
464 if jump_args_count == 0 {
465 // No jump arguments
466 for (depth, _) in depths.iter() {
467 let block = {
468 let i = state.control_stack.len() - 1 - (*depth as usize);
469 let frame = &mut state.control_stack[i];
470 frame.set_branched_to_exit();
471 frame.br_destination()
472 };
473 data.push_entry(block);
474 }
475 let jt = builder.create_jump_table(data);
476 let block = {
477 let i = state.control_stack.len() - 1 - (default as usize);
478 let frame = &mut state.control_stack[i];
479 frame.set_branched_to_exit();
480 frame.br_destination()
481 };
482 builder.ins().br_table(val, block, jt);
483 } else {
484 // Here we have jump arguments, but Cranelift's br_table doesn't support them
485 // We then proceed to split the edges going out of the br_table
486 let return_count = jump_args_count;
487 let mut dest_block_sequence = vec![];
488 let mut dest_block_map = HashMap::new();
489 for (depth, _) in depths.iter() {
490 let branch_block = match dest_block_map.entry(*depth as usize) {
491 hash_map::Entry::Occupied(entry) => *entry.get(),
492 hash_map::Entry::Vacant(entry) => {
493 let block = builder.create_block();
494 dest_block_sequence.push((*depth as usize, block));
495 *entry.insert(block)
496 }
497 };
498 data.push_entry(branch_block);
499 }
500 let default_branch_block = match dest_block_map.entry(default as usize) {
501 hash_map::Entry::Occupied(entry) => *entry.get(),
502 hash_map::Entry::Vacant(entry) => {
503 let block = builder.create_block();
504 dest_block_sequence.push((default as usize, block));
505 *entry.insert(block)
506 }
507 };
508 let jt = builder.create_jump_table(data);
509 builder.ins().br_table(val, default_branch_block, jt);
510 for (depth, dest_block) in dest_block_sequence {
511 builder.switch_to_block(dest_block);
512 builder.seal_block(dest_block);
513 let real_dest_block = {
514 let i = state.control_stack.len() - 1 - depth;
515 let frame = &mut state.control_stack[i];
516 frame.set_branched_to_exit();
517 frame.br_destination()
518 };
519 let destination_args = state.peekn_mut(return_count);
520 canonicalise_then_jump(builder, real_dest_block, destination_args);
521 }
522 state.popn(return_count);
523 }
524 state.reachable = false;
525 }
526 Operator::Return => {
527 let (return_count, br_destination) = {
528 let frame = &mut state.control_stack[0];
529 if environ.return_mode() == ReturnMode::FallthroughReturn {
530 frame.set_branched_to_exit();
531 }
532 let return_count = frame.num_return_values();
533 (return_count, frame.br_destination())
534 };
535 {
536 let return_args = state.peekn_mut(return_count);
537 let return_types = wasm_param_types(&builder.func.signature.returns, |i| {
538 environ.is_wasm_return(&builder.func.signature, i)
539 });
540 bitcast_arguments(return_args, &return_types, builder);
541 match environ.return_mode() {
542 ReturnMode::NormalReturns => builder.ins().return_(return_args),
543 ReturnMode::FallthroughReturn => {
544 canonicalise_then_jump(builder, br_destination, return_args)
545 }
546 };
547 }
548 state.popn(return_count);
549 state.reachable = false;
550 }
551 /********************************** Exception handing **********************************/
552 Operator::Try { .. }
553 | Operator::Catch { .. }
554 | Operator::Throw { .. }
555 | Operator::Unwind
556 | Operator::Rethrow { .. }
557 | Operator::Delegate { .. }
558 | Operator::CatchAll => {
559 return Err(wasm_unsupported!(
560 "proposed exception handling operator {:?}",
561 op
562 ));
563 }
564 /************************************ Calls ****************************************
565 * The call instructions pop off their arguments from the stack and append their
566 * return values to it. `call_indirect` needs environment support because there is an
567 * argument referring to an index in the external functions table of the module.
568 ************************************************************************************/
569 Operator::Call { function_index } => {
570 let (fref, num_args) = state.get_direct_func(builder.func, *function_index, environ)?;
571
572 // Bitcast any vector arguments to their default type, I8X16, before calling.
573 let callee_signature =
574 &builder.func.dfg.signatures[builder.func.dfg.ext_funcs[fref].signature];
575 let args = state.peekn_mut(num_args);
576 let types = wasm_param_types(&callee_signature.params, |i| {
577 environ.is_wasm_parameter(&callee_signature, i)
578 });
579 bitcast_arguments(args, &types, builder);
580
581 let call = environ.translate_call(
582 builder.cursor(),
583 FuncIndex::from_u32(*function_index),
584 fref,
585 args,
586 )?;
587 let inst_results = builder.inst_results(call);
588 debug_assert_eq!(
589 inst_results.len(),
590 builder.func.dfg.signatures[builder.func.dfg.ext_funcs[fref].signature]
591 .returns
592 .len(),
593 "translate_call results should match the call signature"
594 );
595 state.popn(num_args);
596 state.pushn(inst_results);
597 }
598 Operator::CallIndirect { index, table_index } => {
599 // `index` is the index of the function's signature and `table_index` is the index of
600 // the table to search the function in.
601 let (sigref, num_args) = state.get_indirect_sig(builder.func, *index, environ)?;
602 let table = state.get_or_create_table(builder.func, *table_index, environ)?;
603 let callee = state.pop1();
604
605 // Bitcast any vector arguments to their default type, I8X16, before calling.
606 let callee_signature = &builder.func.dfg.signatures[sigref];
607 let args = state.peekn_mut(num_args);
608 let types = wasm_param_types(&callee_signature.params, |i| {
609 environ.is_wasm_parameter(&callee_signature, i)
610 });
611 bitcast_arguments(args, &types, builder);
612
613 let call = environ.translate_call_indirect(
614 builder.cursor(),
615 TableIndex::from_u32(*table_index),
616 table,
617 TypeIndex::from_u32(*index),
618 sigref,
619 callee,
620 state.peekn(num_args),
621 )?;
622 let inst_results = builder.inst_results(call);
623 debug_assert_eq!(
624 inst_results.len(),
625 builder.func.dfg.signatures[sigref].returns.len(),
626 "translate_call_indirect results should match the call signature"
627 );
628 state.popn(num_args);
629 state.pushn(inst_results);
630 }
631 /******************************* Memory management ***********************************
632 * Memory management is handled by environment. It is usually translated into calls to
633 * special functions.
634 ************************************************************************************/
635 Operator::MemoryGrow { mem, mem_byte: _ } => {
636 // The WebAssembly MVP only supports one linear memory, but we expect the reserved
637 // argument to be a memory index.
638 let heap_index = MemoryIndex::from_u32(*mem);
639 let heap = state.get_heap(builder.func, *mem, environ)?;
640 let val = state.pop1();
641 state.push1(environ.translate_memory_grow(builder.cursor(), heap_index, heap, val)?)
642 }
643 Operator::MemorySize { mem, mem_byte: _ } => {
644 let heap_index = MemoryIndex::from_u32(*mem);
645 let heap = state.get_heap(builder.func, *mem, environ)?;
646 state.push1(environ.translate_memory_size(builder.cursor(), heap_index, heap)?);
647 }
648 /******************************* Load instructions ***********************************
649 * Wasm specifies an integer alignment flag but we drop it in Cranelift.
650 * The memory base address is provided by the environment.
651 ************************************************************************************/
652 Operator::I32Load8U { memarg } => {
653 translate_load(memarg, ir::Opcode::Uload8, I32, builder, state, environ)?;
654 }
655 Operator::I32Load16U { memarg } => {
656 translate_load(memarg, ir::Opcode::Uload16, I32, builder, state, environ)?;
657 }
658 Operator::I32Load8S { memarg } => {
659 translate_load(memarg, ir::Opcode::Sload8, I32, builder, state, environ)?;
660 }
661 Operator::I32Load16S { memarg } => {
662 translate_load(memarg, ir::Opcode::Sload16, I32, builder, state, environ)?;
663 }
664 Operator::I64Load8U { memarg } => {
665 translate_load(memarg, ir::Opcode::Uload8, I64, builder, state, environ)?;
666 }
667 Operator::I64Load16U { memarg } => {
668 translate_load(memarg, ir::Opcode::Uload16, I64, builder, state, environ)?;
669 }
670 Operator::I64Load8S { memarg } => {
671 translate_load(memarg, ir::Opcode::Sload8, I64, builder, state, environ)?;
672 }
673 Operator::I64Load16S { memarg } => {
674 translate_load(memarg, ir::Opcode::Sload16, I64, builder, state, environ)?;
675 }
676 Operator::I64Load32S { memarg } => {
677 translate_load(memarg, ir::Opcode::Sload32, I64, builder, state, environ)?;
678 }
679 Operator::I64Load32U { memarg } => {
680 translate_load(memarg, ir::Opcode::Uload32, I64, builder, state, environ)?;
681 }
682 Operator::I32Load { memarg } => {
683 translate_load(memarg, ir::Opcode::Load, I32, builder, state, environ)?;
684 }
685 Operator::F32Load { memarg } => {
686 translate_load(memarg, ir::Opcode::Load, F32, builder, state, environ)?;
687 }
688 Operator::I64Load { memarg } => {
689 translate_load(memarg, ir::Opcode::Load, I64, builder, state, environ)?;
690 }
691 Operator::F64Load { memarg } => {
692 translate_load(memarg, ir::Opcode::Load, F64, builder, state, environ)?;
693 }
694 Operator::V128Load { memarg } => {
695 translate_load(memarg, ir::Opcode::Load, I8X16, builder, state, environ)?;
696 }
697 Operator::V128Load8x8S { memarg } => {
698 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
699 let loaded = builder.ins().sload8x8(flags, base, offset);
700 state.push1(loaded);
701 }
702 Operator::V128Load8x8U { memarg } => {
703 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
704 let loaded = builder.ins().uload8x8(flags, base, offset);
705 state.push1(loaded);
706 }
707 Operator::V128Load16x4S { memarg } => {
708 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
709 let loaded = builder.ins().sload16x4(flags, base, offset);
710 state.push1(loaded);
711 }
712 Operator::V128Load16x4U { memarg } => {
713 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
714 let loaded = builder.ins().uload16x4(flags, base, offset);
715 state.push1(loaded);
716 }
717 Operator::V128Load32x2S { memarg } => {
718 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
719 let loaded = builder.ins().sload32x2(flags, base, offset);
720 state.push1(loaded);
721 }
722 Operator::V128Load32x2U { memarg } => {
723 let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?;
724 let loaded = builder.ins().uload32x2(flags, base, offset);
725 state.push1(loaded);
726 }
727 /****************************** Store instructions ***********************************
728 * Wasm specifies an integer alignment flag but we drop it in Cranelift.
729 * The memory base address is provided by the environment.
730 ************************************************************************************/
731 Operator::I32Store { memarg }
732 | Operator::I64Store { memarg }
733 | Operator::F32Store { memarg }
734 | Operator::F64Store { memarg } => {
735 translate_store(memarg, ir::Opcode::Store, builder, state, environ)?;
736 }
737 Operator::I32Store8 { memarg } | Operator::I64Store8 { memarg } => {
738 translate_store(memarg, ir::Opcode::Istore8, builder, state, environ)?;
739 }
740 Operator::I32Store16 { memarg } | Operator::I64Store16 { memarg } => {
741 translate_store(memarg, ir::Opcode::Istore16, builder, state, environ)?;
742 }
743 Operator::I64Store32 { memarg } => {
744 translate_store(memarg, ir::Opcode::Istore32, builder, state, environ)?;
745 }
746 Operator::V128Store { memarg } => {
747 translate_store(memarg, ir::Opcode::Store, builder, state, environ)?;
748 }
749 /****************************** Nullary Operators ************************************/
750 Operator::I32Const { value } => state.push1(builder.ins().iconst(I32, i64::from(*value))),
751 Operator::I64Const { value } => state.push1(builder.ins().iconst(I64, *value)),
752 Operator::F32Const { value } => {
753 state.push1(builder.ins().f32const(f32_translation(*value)));
754 }
755 Operator::F64Const { value } => {
756 state.push1(builder.ins().f64const(f64_translation(*value)));
757 }
758 /******************************* Unary Operators *************************************/
759 Operator::I32Clz | Operator::I64Clz => {
760 let arg = state.pop1();
761 state.push1(builder.ins().clz(arg));
762 }
763 Operator::I32Ctz | Operator::I64Ctz => {
764 let arg = state.pop1();
765 state.push1(builder.ins().ctz(arg));
766 }
767 Operator::I32Popcnt | Operator::I64Popcnt => {
768 let arg = state.pop1();
769 state.push1(builder.ins().popcnt(arg));
770 }
771 Operator::I64ExtendI32S => {
772 let val = state.pop1();
773 state.push1(builder.ins().sextend(I64, val));
774 }
775 Operator::I64ExtendI32U => {
776 let val = state.pop1();
777 state.push1(builder.ins().uextend(I64, val));
778 }
779 Operator::I32WrapI64 => {
780 let val = state.pop1();
781 state.push1(builder.ins().ireduce(I32, val));
782 }
783 Operator::F32Sqrt | Operator::F64Sqrt => {
784 let arg = state.pop1();
785 state.push1(builder.ins().sqrt(arg));
786 }
787 Operator::F32Ceil | Operator::F64Ceil => {
788 let arg = state.pop1();
789 state.push1(builder.ins().ceil(arg));
790 }
791 Operator::F32Floor | Operator::F64Floor => {
792 let arg = state.pop1();
793 state.push1(builder.ins().floor(arg));
794 }
795 Operator::F32Trunc | Operator::F64Trunc => {
796 let arg = state.pop1();
797 state.push1(builder.ins().trunc(arg));
798 }
799 Operator::F32Nearest | Operator::F64Nearest => {
800 let arg = state.pop1();
801 state.push1(builder.ins().nearest(arg));
802 }
803 Operator::F32Abs | Operator::F64Abs => {
804 let val = state.pop1();
805 state.push1(builder.ins().fabs(val));
806 }
807 Operator::F32Neg | Operator::F64Neg => {
808 let arg = state.pop1();
809 state.push1(builder.ins().fneg(arg));
810 }
811 Operator::F64ConvertI64U | Operator::F64ConvertI32U => {
812 let val = state.pop1();
813 state.push1(builder.ins().fcvt_from_uint(F64, val));
814 }
815 Operator::F64ConvertI64S | Operator::F64ConvertI32S => {
816 let val = state.pop1();
817 state.push1(builder.ins().fcvt_from_sint(F64, val));
818 }
819 Operator::F32ConvertI64S | Operator::F32ConvertI32S => {
820 let val = state.pop1();
821 state.push1(builder.ins().fcvt_from_sint(F32, val));
822 }
823 Operator::F32ConvertI64U | Operator::F32ConvertI32U => {
824 let val = state.pop1();
825 state.push1(builder.ins().fcvt_from_uint(F32, val));
826 }
827 Operator::F64PromoteF32 => {
828 let val = state.pop1();
829 state.push1(builder.ins().fpromote(F64, val));
830 }
831 Operator::F32DemoteF64 => {
832 let val = state.pop1();
833 state.push1(builder.ins().fdemote(F32, val));
834 }
835 Operator::I64TruncF64S | Operator::I64TruncF32S => {
836 let val = state.pop1();
837 state.push1(builder.ins().fcvt_to_sint(I64, val));
838 }
839 Operator::I32TruncF64S | Operator::I32TruncF32S => {
840 let val = state.pop1();
841 state.push1(builder.ins().fcvt_to_sint(I32, val));
842 }
843 Operator::I64TruncF64U | Operator::I64TruncF32U => {
844 let val = state.pop1();
845 state.push1(builder.ins().fcvt_to_uint(I64, val));
846 }
847 Operator::I32TruncF64U | Operator::I32TruncF32U => {
848 let val = state.pop1();
849 state.push1(builder.ins().fcvt_to_uint(I32, val));
850 }
851 Operator::I64TruncSatF64S | Operator::I64TruncSatF32S => {
852 let val = state.pop1();
853 state.push1(builder.ins().fcvt_to_sint_sat(I64, val));
854 }
855 Operator::I32TruncSatF64S | Operator::I32TruncSatF32S => {
856 let val = state.pop1();
857 state.push1(builder.ins().fcvt_to_sint_sat(I32, val));
858 }
859 Operator::I64TruncSatF64U | Operator::I64TruncSatF32U => {
860 let val = state.pop1();
861 state.push1(builder.ins().fcvt_to_uint_sat(I64, val));
862 }
863 Operator::I32TruncSatF64U | Operator::I32TruncSatF32U => {
864 let val = state.pop1();
865 state.push1(builder.ins().fcvt_to_uint_sat(I32, val));
866 }
867 Operator::F32ReinterpretI32 => {
868 let val = state.pop1();
869 state.push1(builder.ins().bitcast(F32, val));
870 }
871 Operator::F64ReinterpretI64 => {
872 let val = state.pop1();
873 state.push1(builder.ins().bitcast(F64, val));
874 }
875 Operator::I32ReinterpretF32 => {
876 let val = state.pop1();
877 state.push1(builder.ins().bitcast(I32, val));
878 }
879 Operator::I64ReinterpretF64 => {
880 let val = state.pop1();
881 state.push1(builder.ins().bitcast(I64, val));
882 }
883 Operator::I32Extend8S => {
884 let val = state.pop1();
885 state.push1(builder.ins().ireduce(I8, val));
886 let val = state.pop1();
887 state.push1(builder.ins().sextend(I32, val));
888 }
889 Operator::I32Extend16S => {
890 let val = state.pop1();
891 state.push1(builder.ins().ireduce(I16, val));
892 let val = state.pop1();
893 state.push1(builder.ins().sextend(I32, val));
894 }
895 Operator::I64Extend8S => {
896 let val = state.pop1();
897 state.push1(builder.ins().ireduce(I8, val));
898 let val = state.pop1();
899 state.push1(builder.ins().sextend(I64, val));
900 }
901 Operator::I64Extend16S => {
902 let val = state.pop1();
903 state.push1(builder.ins().ireduce(I16, val));
904 let val = state.pop1();
905 state.push1(builder.ins().sextend(I64, val));
906 }
907 Operator::I64Extend32S => {
908 let val = state.pop1();
909 state.push1(builder.ins().ireduce(I32, val));
910 let val = state.pop1();
911 state.push1(builder.ins().sextend(I64, val));
912 }
913 /****************************** Binary Operators ************************************/
914 Operator::I32Add | Operator::I64Add => {
915 let (arg1, arg2) = state.pop2();
916 state.push1(builder.ins().iadd(arg1, arg2));
917 }
918 Operator::I32And | Operator::I64And => {
919 let (arg1, arg2) = state.pop2();
920 state.push1(builder.ins().band(arg1, arg2));
921 }
922 Operator::I32Or | Operator::I64Or => {
923 let (arg1, arg2) = state.pop2();
924 state.push1(builder.ins().bor(arg1, arg2));
925 }
926 Operator::I32Xor | Operator::I64Xor => {
927 let (arg1, arg2) = state.pop2();
928 state.push1(builder.ins().bxor(arg1, arg2));
929 }
930 Operator::I32Shl | Operator::I64Shl => {
931 let (arg1, arg2) = state.pop2();
932 state.push1(builder.ins().ishl(arg1, arg2));
933 }
934 Operator::I32ShrS | Operator::I64ShrS => {
935 let (arg1, arg2) = state.pop2();
936 state.push1(builder.ins().sshr(arg1, arg2));
937 }
938 Operator::I32ShrU | Operator::I64ShrU => {
939 let (arg1, arg2) = state.pop2();
940 state.push1(builder.ins().ushr(arg1, arg2));
941 }
942 Operator::I32Rotl | Operator::I64Rotl => {
943 let (arg1, arg2) = state.pop2();
944 state.push1(builder.ins().rotl(arg1, arg2));
945 }
946 Operator::I32Rotr | Operator::I64Rotr => {
947 let (arg1, arg2) = state.pop2();
948 state.push1(builder.ins().rotr(arg1, arg2));
949 }
950 Operator::F32Add | Operator::F64Add => {
951 let (arg1, arg2) = state.pop2();
952 state.push1(builder.ins().fadd(arg1, arg2));
953 }
954 Operator::I32Sub | Operator::I64Sub => {
955 let (arg1, arg2) = state.pop2();
956 state.push1(builder.ins().isub(arg1, arg2));
957 }
958 Operator::F32Sub | Operator::F64Sub => {
959 let (arg1, arg2) = state.pop2();
960 state.push1(builder.ins().fsub(arg1, arg2));
961 }
962 Operator::I32Mul | Operator::I64Mul => {
963 let (arg1, arg2) = state.pop2();
964 state.push1(builder.ins().imul(arg1, arg2));
965 }
966 Operator::F32Mul | Operator::F64Mul => {
967 let (arg1, arg2) = state.pop2();
968 state.push1(builder.ins().fmul(arg1, arg2));
969 }
970 Operator::F32Div | Operator::F64Div => {
971 let (arg1, arg2) = state.pop2();
972 state.push1(builder.ins().fdiv(arg1, arg2));
973 }
974 Operator::I32DivS | Operator::I64DivS => {
975 let (arg1, arg2) = state.pop2();
976 state.push1(builder.ins().sdiv(arg1, arg2));
977 }
978 Operator::I32DivU | Operator::I64DivU => {
979 let (arg1, arg2) = state.pop2();
980 state.push1(builder.ins().udiv(arg1, arg2));
981 }
982 Operator::I32RemS | Operator::I64RemS => {
983 let (arg1, arg2) = state.pop2();
984 state.push1(builder.ins().srem(arg1, arg2));
985 }
986 Operator::I32RemU | Operator::I64RemU => {
987 let (arg1, arg2) = state.pop2();
988 state.push1(builder.ins().urem(arg1, arg2));
989 }
990 Operator::F32Min | Operator::F64Min => {
991 let (arg1, arg2) = state.pop2();
992 state.push1(builder.ins().fmin(arg1, arg2));
993 }
994 Operator::F32Max | Operator::F64Max => {
995 let (arg1, arg2) = state.pop2();
996 state.push1(builder.ins().fmax(arg1, arg2));
997 }
998 Operator::F32Copysign | Operator::F64Copysign => {
999 let (arg1, arg2) = state.pop2();
1000 state.push1(builder.ins().fcopysign(arg1, arg2));
1001 }
1002 /**************************** Comparison Operators **********************************/
1003 Operator::I32LtS | Operator::I64LtS => {
1004 translate_icmp(IntCC::SignedLessThan, builder, state)
1005 }
1006 Operator::I32LtU | Operator::I64LtU => {
1007 translate_icmp(IntCC::UnsignedLessThan, builder, state)
1008 }
1009 Operator::I32LeS | Operator::I64LeS => {
1010 translate_icmp(IntCC::SignedLessThanOrEqual, builder, state)
1011 }
1012 Operator::I32LeU | Operator::I64LeU => {
1013 translate_icmp(IntCC::UnsignedLessThanOrEqual, builder, state)
1014 }
1015 Operator::I32GtS | Operator::I64GtS => {
1016 translate_icmp(IntCC::SignedGreaterThan, builder, state)
1017 }
1018 Operator::I32GtU | Operator::I64GtU => {
1019 translate_icmp(IntCC::UnsignedGreaterThan, builder, state)
1020 }
1021 Operator::I32GeS | Operator::I64GeS => {
1022 translate_icmp(IntCC::SignedGreaterThanOrEqual, builder, state)
1023 }
1024 Operator::I32GeU | Operator::I64GeU => {
1025 translate_icmp(IntCC::UnsignedGreaterThanOrEqual, builder, state)
1026 }
1027 Operator::I32Eqz | Operator::I64Eqz => {
1028 let arg = state.pop1();
1029 let val = builder.ins().icmp_imm(IntCC::Equal, arg, 0);
1030 state.push1(builder.ins().bint(I32, val));
1031 }
1032 Operator::I32Eq | Operator::I64Eq => translate_icmp(IntCC::Equal, builder, state),
1033 Operator::F32Eq | Operator::F64Eq => translate_fcmp(FloatCC::Equal, builder, state),
1034 Operator::I32Ne | Operator::I64Ne => translate_icmp(IntCC::NotEqual, builder, state),
1035 Operator::F32Ne | Operator::F64Ne => translate_fcmp(FloatCC::NotEqual, builder, state),
1036 Operator::F32Gt | Operator::F64Gt => translate_fcmp(FloatCC::GreaterThan, builder, state),
1037 Operator::F32Ge | Operator::F64Ge => {
1038 translate_fcmp(FloatCC::GreaterThanOrEqual, builder, state)
1039 }
1040 Operator::F32Lt | Operator::F64Lt => translate_fcmp(FloatCC::LessThan, builder, state),
1041 Operator::F32Le | Operator::F64Le => {
1042 translate_fcmp(FloatCC::LessThanOrEqual, builder, state)
1043 }
1044 Operator::RefNull { ty } => {
1045 state.push1(environ.translate_ref_null(builder.cursor(), (*ty).try_into()?)?)
1046 }
1047 Operator::RefIsNull => {
1048 let value = state.pop1();
1049 state.push1(environ.translate_ref_is_null(builder.cursor(), value)?);
1050 }
1051 Operator::RefFunc { function_index } => {
1052 let index = FuncIndex::from_u32(*function_index);
1053 state.push1(environ.translate_ref_func(builder.cursor(), index)?);
1054 }
1055 Operator::MemoryAtomicWait32 { memarg } | Operator::MemoryAtomicWait64 { memarg } => {
1056 // The WebAssembly MVP only supports one linear memory and
1057 // wasmparser will ensure that the memory indices specified are
1058 // zero.
1059 let implied_ty = match op {
1060 Operator::MemoryAtomicWait64 { .. } => I64,
1061 Operator::MemoryAtomicWait32 { .. } => I32,
1062 _ => unreachable!(),
1063 };
1064 let heap_index = MemoryIndex::from_u32(memarg.memory);
1065 let heap = state.get_heap(builder.func, memarg.memory, environ)?;
1066 let timeout = state.pop1(); // 64 (fixed)
1067 let expected = state.pop1(); // 32 or 64 (per the `Ixx` in `IxxAtomicWait`)
1068 let addr = state.pop1(); // 32 (fixed)
1069 let addr = fold_atomic_mem_addr(addr, memarg, implied_ty, builder);
1070 assert!(builder.func.dfg.value_type(expected) == implied_ty);
1071 // `fn translate_atomic_wait` can inspect the type of `expected` to figure out what
1072 // code it needs to generate, if it wants.
1073 let res = environ.translate_atomic_wait(
1074 builder.cursor(),
1075 heap_index,
1076 heap,
1077 addr,
1078 expected,
1079 timeout,
1080 )?;
1081 state.push1(res);
1082 }
1083 Operator::MemoryAtomicNotify { memarg } => {
1084 let heap_index = MemoryIndex::from_u32(memarg.memory);
1085 let heap = state.get_heap(builder.func, memarg.memory, environ)?;
1086 let count = state.pop1(); // 32 (fixed)
1087 let addr = state.pop1(); // 32 (fixed)
1088 let addr = fold_atomic_mem_addr(addr, memarg, I32, builder);
1089 let res =
1090 environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?;
1091 state.push1(res);
1092 }
1093 Operator::I32AtomicLoad { memarg } => {
1094 translate_atomic_load(I32, I32, memarg, builder, state, environ)?
1095 }
1096 Operator::I64AtomicLoad { memarg } => {
1097 translate_atomic_load(I64, I64, memarg, builder, state, environ)?
1098 }
1099 Operator::I32AtomicLoad8U { memarg } => {
1100 translate_atomic_load(I32, I8, memarg, builder, state, environ)?
1101 }
1102 Operator::I32AtomicLoad16U { memarg } => {
1103 translate_atomic_load(I32, I16, memarg, builder, state, environ)?
1104 }
1105 Operator::I64AtomicLoad8U { memarg } => {
1106 translate_atomic_load(I64, I8, memarg, builder, state, environ)?
1107 }
1108 Operator::I64AtomicLoad16U { memarg } => {
1109 translate_atomic_load(I64, I16, memarg, builder, state, environ)?
1110 }
1111 Operator::I64AtomicLoad32U { memarg } => {
1112 translate_atomic_load(I64, I32, memarg, builder, state, environ)?
1113 }
1114
1115 Operator::I32AtomicStore { memarg } => {
1116 translate_atomic_store(I32, memarg, builder, state, environ)?
1117 }
1118 Operator::I64AtomicStore { memarg } => {
1119 translate_atomic_store(I64, memarg, builder, state, environ)?
1120 }
1121 Operator::I32AtomicStore8 { memarg } => {
1122 translate_atomic_store(I8, memarg, builder, state, environ)?
1123 }
1124 Operator::I32AtomicStore16 { memarg } => {
1125 translate_atomic_store(I16, memarg, builder, state, environ)?
1126 }
1127 Operator::I64AtomicStore8 { memarg } => {
1128 translate_atomic_store(I8, memarg, builder, state, environ)?
1129 }
1130 Operator::I64AtomicStore16 { memarg } => {
1131 translate_atomic_store(I16, memarg, builder, state, environ)?
1132 }
1133 Operator::I64AtomicStore32 { memarg } => {
1134 translate_atomic_store(I32, memarg, builder, state, environ)?
1135 }
1136
1137 Operator::I32AtomicRmwAdd { memarg } => {
1138 translate_atomic_rmw(I32, I32, AtomicRmwOp::Add, memarg, builder, state, environ)?
1139 }
1140 Operator::I64AtomicRmwAdd { memarg } => {
1141 translate_atomic_rmw(I64, I64, AtomicRmwOp::Add, memarg, builder, state, environ)?
1142 }
1143 Operator::I32AtomicRmw8AddU { memarg } => {
1144 translate_atomic_rmw(I32, I8, AtomicRmwOp::Add, memarg, builder, state, environ)?
1145 }
1146 Operator::I32AtomicRmw16AddU { memarg } => {
1147 translate_atomic_rmw(I32, I16, AtomicRmwOp::Add, memarg, builder, state, environ)?
1148 }
1149 Operator::I64AtomicRmw8AddU { memarg } => {
1150 translate_atomic_rmw(I64, I8, AtomicRmwOp::Add, memarg, builder, state, environ)?
1151 }
1152 Operator::I64AtomicRmw16AddU { memarg } => {
1153 translate_atomic_rmw(I64, I16, AtomicRmwOp::Add, memarg, builder, state, environ)?
1154 }
1155 Operator::I64AtomicRmw32AddU { memarg } => {
1156 translate_atomic_rmw(I64, I32, AtomicRmwOp::Add, memarg, builder, state, environ)?
1157 }
1158
1159 Operator::I32AtomicRmwSub { memarg } => {
1160 translate_atomic_rmw(I32, I32, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1161 }
1162 Operator::I64AtomicRmwSub { memarg } => {
1163 translate_atomic_rmw(I64, I64, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1164 }
1165 Operator::I32AtomicRmw8SubU { memarg } => {
1166 translate_atomic_rmw(I32, I8, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1167 }
1168 Operator::I32AtomicRmw16SubU { memarg } => {
1169 translate_atomic_rmw(I32, I16, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1170 }
1171 Operator::I64AtomicRmw8SubU { memarg } => {
1172 translate_atomic_rmw(I64, I8, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1173 }
1174 Operator::I64AtomicRmw16SubU { memarg } => {
1175 translate_atomic_rmw(I64, I16, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1176 }
1177 Operator::I64AtomicRmw32SubU { memarg } => {
1178 translate_atomic_rmw(I64, I32, AtomicRmwOp::Sub, memarg, builder, state, environ)?
1179 }
1180
1181 Operator::I32AtomicRmwAnd { memarg } => {
1182 translate_atomic_rmw(I32, I32, AtomicRmwOp::And, memarg, builder, state, environ)?
1183 }
1184 Operator::I64AtomicRmwAnd { memarg } => {
1185 translate_atomic_rmw(I64, I64, AtomicRmwOp::And, memarg, builder, state, environ)?
1186 }
1187 Operator::I32AtomicRmw8AndU { memarg } => {
1188 translate_atomic_rmw(I32, I8, AtomicRmwOp::And, memarg, builder, state, environ)?
1189 }
1190 Operator::I32AtomicRmw16AndU { memarg } => {
1191 translate_atomic_rmw(I32, I16, AtomicRmwOp::And, memarg, builder, state, environ)?
1192 }
1193 Operator::I64AtomicRmw8AndU { memarg } => {
1194 translate_atomic_rmw(I64, I8, AtomicRmwOp::And, memarg, builder, state, environ)?
1195 }
1196 Operator::I64AtomicRmw16AndU { memarg } => {
1197 translate_atomic_rmw(I64, I16, AtomicRmwOp::And, memarg, builder, state, environ)?
1198 }
1199 Operator::I64AtomicRmw32AndU { memarg } => {
1200 translate_atomic_rmw(I64, I32, AtomicRmwOp::And, memarg, builder, state, environ)?
1201 }
1202
1203 Operator::I32AtomicRmwOr { memarg } => {
1204 translate_atomic_rmw(I32, I32, AtomicRmwOp::Or, memarg, builder, state, environ)?
1205 }
1206 Operator::I64AtomicRmwOr { memarg } => {
1207 translate_atomic_rmw(I64, I64, AtomicRmwOp::Or, memarg, builder, state, environ)?
1208 }
1209 Operator::I32AtomicRmw8OrU { memarg } => {
1210 translate_atomic_rmw(I32, I8, AtomicRmwOp::Or, memarg, builder, state, environ)?
1211 }
1212 Operator::I32AtomicRmw16OrU { memarg } => {
1213 translate_atomic_rmw(I32, I16, AtomicRmwOp::Or, memarg, builder, state, environ)?
1214 }
1215 Operator::I64AtomicRmw8OrU { memarg } => {
1216 translate_atomic_rmw(I64, I8, AtomicRmwOp::Or, memarg, builder, state, environ)?
1217 }
1218 Operator::I64AtomicRmw16OrU { memarg } => {
1219 translate_atomic_rmw(I64, I16, AtomicRmwOp::Or, memarg, builder, state, environ)?
1220 }
1221 Operator::I64AtomicRmw32OrU { memarg } => {
1222 translate_atomic_rmw(I64, I32, AtomicRmwOp::Or, memarg, builder, state, environ)?
1223 }
1224
1225 Operator::I32AtomicRmwXor { memarg } => {
1226 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1227 }
1228 Operator::I64AtomicRmwXor { memarg } => {
1229 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1230 }
1231 Operator::I32AtomicRmw8XorU { memarg } => {
1232 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1233 }
1234 Operator::I32AtomicRmw16XorU { memarg } => {
1235 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1236 }
1237 Operator::I64AtomicRmw8XorU { memarg } => {
1238 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1239 }
1240 Operator::I64AtomicRmw16XorU { memarg } => {
1241 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1242 }
1243 Operator::I64AtomicRmw32XorU { memarg } => {
1244 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xor, memarg, builder, state, environ)?
1245 }
1246
1247 Operator::I32AtomicRmwXchg { memarg } => {
1248 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1249 }
1250 Operator::I64AtomicRmwXchg { memarg } => {
1251 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1252 }
1253 Operator::I32AtomicRmw8XchgU { memarg } => {
1254 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1255 }
1256 Operator::I32AtomicRmw16XchgU { memarg } => {
1257 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1258 }
1259 Operator::I64AtomicRmw8XchgU { memarg } => {
1260 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1261 }
1262 Operator::I64AtomicRmw16XchgU { memarg } => {
1263 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1264 }
1265 Operator::I64AtomicRmw32XchgU { memarg } => {
1266 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xchg, memarg, builder, state, environ)?
1267 }
1268
1269 Operator::I32AtomicRmwCmpxchg { memarg } => {
1270 translate_atomic_cas(I32, I32, memarg, builder, state, environ)?
1271 }
1272 Operator::I64AtomicRmwCmpxchg { memarg } => {
1273 translate_atomic_cas(I64, I64, memarg, builder, state, environ)?
1274 }
1275 Operator::I32AtomicRmw8CmpxchgU { memarg } => {
1276 translate_atomic_cas(I32, I8, memarg, builder, state, environ)?
1277 }
1278 Operator::I32AtomicRmw16CmpxchgU { memarg } => {
1279 translate_atomic_cas(I32, I16, memarg, builder, state, environ)?
1280 }
1281 Operator::I64AtomicRmw8CmpxchgU { memarg } => {
1282 translate_atomic_cas(I64, I8, memarg, builder, state, environ)?
1283 }
1284 Operator::I64AtomicRmw16CmpxchgU { memarg } => {
1285 translate_atomic_cas(I64, I16, memarg, builder, state, environ)?
1286 }
1287 Operator::I64AtomicRmw32CmpxchgU { memarg } => {
1288 translate_atomic_cas(I64, I32, memarg, builder, state, environ)?
1289 }
1290
1291 Operator::AtomicFence { .. } => {
1292 builder.ins().fence();
1293 }
1294 Operator::MemoryCopy { src, dst } => {
1295 let src_index = MemoryIndex::from_u32(*src);
1296 let dst_index = MemoryIndex::from_u32(*dst);
1297 let src_heap = state.get_heap(builder.func, *src, environ)?;
1298 let dst_heap = state.get_heap(builder.func, *dst, environ)?;
1299 let len = state.pop1();
1300 let src_pos = state.pop1();
1301 let dst_pos = state.pop1();
1302 environ.translate_memory_copy(
1303 builder.cursor(),
1304 src_index,
1305 src_heap,
1306 dst_index,
1307 dst_heap,
1308 dst_pos,
1309 src_pos,
1310 len,
1311 )?;
1312 }
1313 Operator::MemoryFill { mem } => {
1314 let heap_index = MemoryIndex::from_u32(*mem);
1315 let heap = state.get_heap(builder.func, *mem, environ)?;
1316 let len = state.pop1();
1317 let val = state.pop1();
1318 let dest = state.pop1();
1319 environ.translate_memory_fill(builder.cursor(), heap_index, heap, dest, val, len)?;
1320 }
1321 Operator::MemoryInit { segment, mem } => {
1322 let heap_index = MemoryIndex::from_u32(*mem);
1323 let heap = state.get_heap(builder.func, *mem, environ)?;
1324 let len = state.pop1();
1325 let src = state.pop1();
1326 let dest = state.pop1();
1327 environ.translate_memory_init(
1328 builder.cursor(),
1329 heap_index,
1330 heap,
1331 *segment,
1332 dest,
1333 src,
1334 len,
1335 )?;
1336 }
1337 Operator::DataDrop { segment } => {
1338 environ.translate_data_drop(builder.cursor(), *segment)?;
1339 }
1340 Operator::TableSize { table: index } => {
1341 let table = state.get_or_create_table(builder.func, *index, environ)?;
1342 state.push1(environ.translate_table_size(
1343 builder.cursor(),
1344 TableIndex::from_u32(*index),
1345 table,
1346 )?);
1347 }
1348 Operator::TableGrow { table: index } => {
1349 let table_index = TableIndex::from_u32(*index);
1350 let table = state.get_or_create_table(builder.func, *index, environ)?;
1351 let delta = state.pop1();
1352 let init_value = state.pop1();
1353 state.push1(environ.translate_table_grow(
1354 builder.cursor(),
1355 table_index,
1356 table,
1357 delta,
1358 init_value,
1359 )?);
1360 }
1361 Operator::TableGet { table: index } => {
1362 let table_index = TableIndex::from_u32(*index);
1363 let table = state.get_or_create_table(builder.func, *index, environ)?;
1364 let index = state.pop1();
1365 state.push1(environ.translate_table_get(builder, table_index, table, index)?);
1366 }
1367 Operator::TableSet { table: index } => {
1368 let table_index = TableIndex::from_u32(*index);
1369 let table = state.get_or_create_table(builder.func, *index, environ)?;
1370 let value = state.pop1();
1371 let index = state.pop1();
1372 environ.translate_table_set(builder, table_index, table, value, index)?;
1373 }
1374 Operator::TableCopy {
1375 dst_table: dst_table_index,
1376 src_table: src_table_index,
1377 } => {
1378 let dst_table = state.get_or_create_table(builder.func, *dst_table_index, environ)?;
1379 let src_table = state.get_or_create_table(builder.func, *src_table_index, environ)?;
1380 let len = state.pop1();
1381 let src = state.pop1();
1382 let dest = state.pop1();
1383 environ.translate_table_copy(
1384 builder.cursor(),
1385 TableIndex::from_u32(*dst_table_index),
1386 dst_table,
1387 TableIndex::from_u32(*src_table_index),
1388 src_table,
1389 dest,
1390 src,
1391 len,
1392 )?;
1393 }
1394 Operator::TableFill { table } => {
1395 let table_index = TableIndex::from_u32(*table);
1396 let len = state.pop1();
1397 let val = state.pop1();
1398 let dest = state.pop1();
1399 environ.translate_table_fill(builder.cursor(), table_index, dest, val, len)?;
1400 }
1401 Operator::TableInit {
1402 segment,
1403 table: table_index,
1404 } => {
1405 let table = state.get_or_create_table(builder.func, *table_index, environ)?;
1406 let len = state.pop1();
1407 let src = state.pop1();
1408 let dest = state.pop1();
1409 environ.translate_table_init(
1410 builder.cursor(),
1411 *segment,
1412 TableIndex::from_u32(*table_index),
1413 table,
1414 dest,
1415 src,
1416 len,
1417 )?;
1418 }
1419 Operator::ElemDrop { segment } => {
1420 environ.translate_elem_drop(builder.cursor(), *segment)?;
1421 }
1422 Operator::V128Const { value } => {
1423 let data = value.bytes().to_vec().into();
1424 let handle = builder.func.dfg.constants.insert(data);
1425 let value = builder.ins().vconst(I8X16, handle);
1426 // the v128.const is typed in CLIF as a I8x16 but raw_bitcast to a different type
1427 // before use
1428 state.push1(value)
1429 }
1430 Operator::I8x16Splat | Operator::I16x8Splat => {
1431 let reduced = builder.ins().ireduce(type_of(op).lane_type(), state.pop1());
1432 let splatted = builder.ins().splat(type_of(op), reduced);
1433 state.push1(splatted)
1434 }
1435 Operator::I32x4Splat
1436 | Operator::I64x2Splat
1437 | Operator::F32x4Splat
1438 | Operator::F64x2Splat => {
1439 let splatted = builder.ins().splat(type_of(op), state.pop1());
1440 state.push1(splatted)
1441 }
1442 Operator::V128Load8Splat { memarg }
1443 | Operator::V128Load16Splat { memarg }
1444 | Operator::V128Load32Splat { memarg }
1445 | Operator::V128Load64Splat { memarg } => {
1446 translate_load(
1447 memarg,
1448 ir::Opcode::Load,
1449 type_of(op).lane_type(),
1450 builder,
1451 state,
1452 environ,
1453 )?;
1454 let splatted = builder.ins().splat(type_of(op), state.pop1());
1455 state.push1(splatted)
1456 }
1457 Operator::V128Load32Zero { memarg } | Operator::V128Load64Zero { memarg } => {
1458 translate_load(
1459 memarg,
1460 ir::Opcode::Load,
1461 type_of(op).lane_type(),
1462 builder,
1463 state,
1464 environ,
1465 )?;
1466 let as_vector = builder.ins().scalar_to_vector(type_of(op), state.pop1());
1467 state.push1(as_vector)
1468 }
1469 Operator::V128Load8Lane { memarg, lane }
1470 | Operator::V128Load16Lane { memarg, lane }
1471 | Operator::V128Load32Lane { memarg, lane }
1472 | Operator::V128Load64Lane { memarg, lane } => {
1473 let vector = pop1_with_bitcast(state, type_of(op), builder);
1474 translate_load(
1475 memarg,
1476 ir::Opcode::Load,
1477 type_of(op).lane_type(),
1478 builder,
1479 state,
1480 environ,
1481 )?;
1482 let replacement = state.pop1();
1483 state.push1(builder.ins().insertlane(vector, replacement, *lane))
1484 }
1485 Operator::V128Store8Lane { memarg, lane }
1486 | Operator::V128Store16Lane { memarg, lane }
1487 | Operator::V128Store32Lane { memarg, lane }
1488 | Operator::V128Store64Lane { memarg, lane } => {
1489 let vector = pop1_with_bitcast(state, type_of(op), builder);
1490 state.push1(builder.ins().extractlane(vector, lane.clone()));
1491 translate_store(memarg, ir::Opcode::Store, builder, state, environ)?;
1492 }
1493 Operator::I8x16ExtractLaneS { lane } | Operator::I16x8ExtractLaneS { lane } => {
1494 let vector = pop1_with_bitcast(state, type_of(op), builder);
1495 let extracted = builder.ins().extractlane(vector, lane.clone());
1496 state.push1(builder.ins().sextend(I32, extracted))
1497 }
1498 Operator::I8x16ExtractLaneU { lane } | Operator::I16x8ExtractLaneU { lane } => {
1499 let vector = pop1_with_bitcast(state, type_of(op), builder);
1500 let extracted = builder.ins().extractlane(vector, lane.clone());
1501 state.push1(builder.ins().uextend(I32, extracted));
1502 // On x86, PEXTRB zeroes the upper bits of the destination register of extractlane so
1503 // uextend could be elided; for now, uextend is needed for Cranelift's type checks to
1504 // work.
1505 }
1506 Operator::I32x4ExtractLane { lane }
1507 | Operator::I64x2ExtractLane { lane }
1508 | Operator::F32x4ExtractLane { lane }
1509 | Operator::F64x2ExtractLane { lane } => {
1510 let vector = pop1_with_bitcast(state, type_of(op), builder);
1511 state.push1(builder.ins().extractlane(vector, lane.clone()))
1512 }
1513 Operator::I8x16ReplaceLane { lane } | Operator::I16x8ReplaceLane { lane } => {
1514 let (vector, replacement) = state.pop2();
1515 let ty = type_of(op);
1516 let reduced = builder.ins().ireduce(ty.lane_type(), replacement);
1517 let vector = optionally_bitcast_vector(vector, ty, builder);
1518 state.push1(builder.ins().insertlane(vector, reduced, *lane))
1519 }
1520 Operator::I32x4ReplaceLane { lane }
1521 | Operator::I64x2ReplaceLane { lane }
1522 | Operator::F32x4ReplaceLane { lane }
1523 | Operator::F64x2ReplaceLane { lane } => {
1524 let (vector, replacement) = state.pop2();
1525 let vector = optionally_bitcast_vector(vector, type_of(op), builder);
1526 state.push1(builder.ins().insertlane(vector, replacement, *lane))
1527 }
1528 Operator::I8x16Shuffle { lanes, .. } => {
1529 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
1530 let lanes = ConstantData::from(lanes.as_ref());
1531 let mask = builder.func.dfg.immediates.push(lanes);
1532 let shuffled = builder.ins().shuffle(a, b, mask);
1533 state.push1(shuffled)
1534 // At this point the original types of a and b are lost; users of this value (i.e. this
1535 // WASM-to-CLIF translator) may need to raw_bitcast for type-correctness. This is due
1536 // to WASM using the less specific v128 type for certain operations and more specific
1537 // types (e.g. i8x16) for others.
1538 }
1539 Operator::I8x16Swizzle => {
1540 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
1541 state.push1(builder.ins().swizzle(I8X16, a, b))
1542 }
1543 Operator::I8x16Add | Operator::I16x8Add | Operator::I32x4Add | Operator::I64x2Add => {
1544 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1545 state.push1(builder.ins().iadd(a, b))
1546 }
1547 Operator::I8x16AddSatS | Operator::I16x8AddSatS => {
1548 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1549 state.push1(builder.ins().sadd_sat(a, b))
1550 }
1551 Operator::I8x16AddSatU | Operator::I16x8AddSatU => {
1552 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1553 state.push1(builder.ins().uadd_sat(a, b))
1554 }
1555 Operator::I8x16Sub | Operator::I16x8Sub | Operator::I32x4Sub | Operator::I64x2Sub => {
1556 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1557 state.push1(builder.ins().isub(a, b))
1558 }
1559 Operator::I8x16SubSatS | Operator::I16x8SubSatS => {
1560 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1561 state.push1(builder.ins().ssub_sat(a, b))
1562 }
1563 Operator::I8x16SubSatU | Operator::I16x8SubSatU => {
1564 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1565 state.push1(builder.ins().usub_sat(a, b))
1566 }
1567 Operator::I8x16MinS | Operator::I16x8MinS | Operator::I32x4MinS => {
1568 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1569 state.push1(builder.ins().imin(a, b))
1570 }
1571 Operator::I8x16MinU | Operator::I16x8MinU | Operator::I32x4MinU => {
1572 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1573 state.push1(builder.ins().umin(a, b))
1574 }
1575 Operator::I8x16MaxS | Operator::I16x8MaxS | Operator::I32x4MaxS => {
1576 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1577 state.push1(builder.ins().imax(a, b))
1578 }
1579 Operator::I8x16MaxU | Operator::I16x8MaxU | Operator::I32x4MaxU => {
1580 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1581 state.push1(builder.ins().umax(a, b))
1582 }
1583 Operator::I8x16RoundingAverageU | Operator::I16x8RoundingAverageU => {
1584 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1585 state.push1(builder.ins().avg_round(a, b))
1586 }
1587 Operator::I8x16Neg | Operator::I16x8Neg | Operator::I32x4Neg | Operator::I64x2Neg => {
1588 let a = pop1_with_bitcast(state, type_of(op), builder);
1589 state.push1(builder.ins().ineg(a))
1590 }
1591 Operator::I8x16Abs | Operator::I16x8Abs | Operator::I32x4Abs | Operator::I64x2Abs => {
1592 let a = pop1_with_bitcast(state, type_of(op), builder);
1593 state.push1(builder.ins().iabs(a))
1594 }
1595 Operator::I16x8Mul | Operator::I32x4Mul | Operator::I64x2Mul => {
1596 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1597 state.push1(builder.ins().imul(a, b))
1598 }
1599 Operator::V128Or => {
1600 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1601 state.push1(builder.ins().bor(a, b))
1602 }
1603 Operator::V128Xor => {
1604 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1605 state.push1(builder.ins().bxor(a, b))
1606 }
1607 Operator::V128And => {
1608 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1609 state.push1(builder.ins().band(a, b))
1610 }
1611 Operator::V128AndNot => {
1612 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1613 state.push1(builder.ins().band_not(a, b))
1614 }
1615 Operator::V128Not => {
1616 let a = state.pop1();
1617 state.push1(builder.ins().bnot(a));
1618 }
1619 Operator::I8x16Shl | Operator::I16x8Shl | Operator::I32x4Shl | Operator::I64x2Shl => {
1620 let (a, b) = state.pop2();
1621 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1622 let bitwidth = i64::from(type_of(op).lane_bits());
1623 // The spec expects to shift with `b mod lanewidth`; so, e.g., for 16 bit lane-width
1624 // we do `b AND 15`; this means fewer instructions than `iconst + urem`.
1625 let b_mod_bitwidth = builder.ins().band_imm(b, bitwidth - 1);
1626 state.push1(builder.ins().ishl(bitcast_a, b_mod_bitwidth))
1627 }
1628 Operator::I8x16ShrU | Operator::I16x8ShrU | Operator::I32x4ShrU | Operator::I64x2ShrU => {
1629 let (a, b) = state.pop2();
1630 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1631 let bitwidth = i64::from(type_of(op).lane_bits());
1632 // The spec expects to shift with `b mod lanewidth`; so, e.g., for 16 bit lane-width
1633 // we do `b AND 15`; this means fewer instructions than `iconst + urem`.
1634 let b_mod_bitwidth = builder.ins().band_imm(b, bitwidth - 1);
1635 state.push1(builder.ins().ushr(bitcast_a, b_mod_bitwidth))
1636 }
1637 Operator::I8x16ShrS | Operator::I16x8ShrS | Operator::I32x4ShrS | Operator::I64x2ShrS => {
1638 let (a, b) = state.pop2();
1639 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1640 let bitwidth = i64::from(type_of(op).lane_bits());
1641 // The spec expects to shift with `b mod lanewidth`; so, e.g., for 16 bit lane-width
1642 // we do `b AND 15`; this means fewer instructions than `iconst + urem`.
1643 let b_mod_bitwidth = builder.ins().band_imm(b, bitwidth - 1);
1644 state.push1(builder.ins().sshr(bitcast_a, b_mod_bitwidth))
1645 }
1646 Operator::V128Bitselect => {
1647 let (a, b, c) = state.pop3();
1648 let bitcast_a = optionally_bitcast_vector(a, I8X16, builder);
1649 let bitcast_b = optionally_bitcast_vector(b, I8X16, builder);
1650 let bitcast_c = optionally_bitcast_vector(c, I8X16, builder);
1651 // The CLIF operand ordering is slightly different and the types of all three
1652 // operands must match (hence the bitcast).
1653 state.push1(builder.ins().bitselect(bitcast_c, bitcast_a, bitcast_b))
1654 }
1655 Operator::V128AnyTrue => {
1656 let a = pop1_with_bitcast(state, type_of(op), builder);
1657 let bool_result = builder.ins().vany_true(a);
1658 state.push1(builder.ins().bint(I32, bool_result))
1659 }
1660 Operator::I8x16AllTrue
1661 | Operator::I16x8AllTrue
1662 | Operator::I32x4AllTrue
1663 | Operator::I64x2AllTrue => {
1664 let a = pop1_with_bitcast(state, type_of(op), builder);
1665 let bool_result = builder.ins().vall_true(a);
1666 state.push1(builder.ins().bint(I32, bool_result))
1667 }
1668 Operator::I8x16Bitmask
1669 | Operator::I16x8Bitmask
1670 | Operator::I32x4Bitmask
1671 | Operator::I64x2Bitmask => {
1672 let a = pop1_with_bitcast(state, type_of(op), builder);
1673 state.push1(builder.ins().vhigh_bits(I32, a));
1674 }
1675 Operator::I8x16Eq | Operator::I16x8Eq | Operator::I32x4Eq | Operator::I64x2Eq => {
1676 translate_vector_icmp(IntCC::Equal, type_of(op), builder, state)
1677 }
1678 Operator::I8x16Ne | Operator::I16x8Ne | Operator::I32x4Ne | Operator::I64x2Ne => {
1679 translate_vector_icmp(IntCC::NotEqual, type_of(op), builder, state)
1680 }
1681 Operator::I8x16GtS | Operator::I16x8GtS | Operator::I32x4GtS | Operator::I64x2GtS => {
1682 translate_vector_icmp(IntCC::SignedGreaterThan, type_of(op), builder, state)
1683 }
1684 Operator::I8x16LtS | Operator::I16x8LtS | Operator::I32x4LtS | Operator::I64x2LtS => {
1685 translate_vector_icmp(IntCC::SignedLessThan, type_of(op), builder, state)
1686 }
1687 Operator::I8x16GtU | Operator::I16x8GtU | Operator::I32x4GtU => {
1688 translate_vector_icmp(IntCC::UnsignedGreaterThan, type_of(op), builder, state)
1689 }
1690 Operator::I8x16LtU | Operator::I16x8LtU | Operator::I32x4LtU => {
1691 translate_vector_icmp(IntCC::UnsignedLessThan, type_of(op), builder, state)
1692 }
1693 Operator::I8x16GeS | Operator::I16x8GeS | Operator::I32x4GeS | Operator::I64x2GeS => {
1694 translate_vector_icmp(IntCC::SignedGreaterThanOrEqual, type_of(op), builder, state)
1695 }
1696 Operator::I8x16LeS | Operator::I16x8LeS | Operator::I32x4LeS | Operator::I64x2LeS => {
1697 translate_vector_icmp(IntCC::SignedLessThanOrEqual, type_of(op), builder, state)
1698 }
1699 Operator::I8x16GeU | Operator::I16x8GeU | Operator::I32x4GeU => translate_vector_icmp(
1700 IntCC::UnsignedGreaterThanOrEqual,
1701 type_of(op),
1702 builder,
1703 state,
1704 ),
1705 Operator::I8x16LeU | Operator::I16x8LeU | Operator::I32x4LeU => {
1706 translate_vector_icmp(IntCC::UnsignedLessThanOrEqual, type_of(op), builder, state)
1707 }
1708 Operator::F32x4Eq | Operator::F64x2Eq => {
1709 translate_vector_fcmp(FloatCC::Equal, type_of(op), builder, state)
1710 }
1711 Operator::F32x4Ne | Operator::F64x2Ne => {
1712 translate_vector_fcmp(FloatCC::NotEqual, type_of(op), builder, state)
1713 }
1714 Operator::F32x4Lt | Operator::F64x2Lt => {
1715 translate_vector_fcmp(FloatCC::LessThan, type_of(op), builder, state)
1716 }
1717 Operator::F32x4Gt | Operator::F64x2Gt => {
1718 translate_vector_fcmp(FloatCC::GreaterThan, type_of(op), builder, state)
1719 }
1720 Operator::F32x4Le | Operator::F64x2Le => {
1721 translate_vector_fcmp(FloatCC::LessThanOrEqual, type_of(op), builder, state)
1722 }
1723 Operator::F32x4Ge | Operator::F64x2Ge => {
1724 translate_vector_fcmp(FloatCC::GreaterThanOrEqual, type_of(op), builder, state)
1725 }
1726 Operator::F32x4Add | Operator::F64x2Add => {
1727 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1728 state.push1(builder.ins().fadd(a, b))
1729 }
1730 Operator::F32x4Sub | Operator::F64x2Sub => {
1731 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1732 state.push1(builder.ins().fsub(a, b))
1733 }
1734 Operator::F32x4Mul | Operator::F64x2Mul => {
1735 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1736 state.push1(builder.ins().fmul(a, b))
1737 }
1738 Operator::F32x4Div | Operator::F64x2Div => {
1739 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1740 state.push1(builder.ins().fdiv(a, b))
1741 }
1742 Operator::F32x4Max | Operator::F64x2Max => {
1743 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1744 state.push1(builder.ins().fmax(a, b))
1745 }
1746 Operator::F32x4Min | Operator::F64x2Min => {
1747 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1748 state.push1(builder.ins().fmin(a, b))
1749 }
1750 Operator::F32x4PMax | Operator::F64x2PMax => {
1751 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1752 state.push1(builder.ins().fmax_pseudo(a, b))
1753 }
1754 Operator::F32x4PMin | Operator::F64x2PMin => {
1755 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1756 state.push1(builder.ins().fmin_pseudo(a, b))
1757 }
1758 Operator::F32x4Sqrt | Operator::F64x2Sqrt => {
1759 let a = pop1_with_bitcast(state, type_of(op), builder);
1760 state.push1(builder.ins().sqrt(a))
1761 }
1762 Operator::F32x4Neg | Operator::F64x2Neg => {
1763 let a = pop1_with_bitcast(state, type_of(op), builder);
1764 state.push1(builder.ins().fneg(a))
1765 }
1766 Operator::F32x4Abs | Operator::F64x2Abs => {
1767 let a = pop1_with_bitcast(state, type_of(op), builder);
1768 state.push1(builder.ins().fabs(a))
1769 }
1770 Operator::F32x4ConvertI32x4S => {
1771 let a = pop1_with_bitcast(state, I32X4, builder);
1772 state.push1(builder.ins().fcvt_from_sint(F32X4, a))
1773 }
1774 Operator::F32x4ConvertI32x4U => {
1775 let a = pop1_with_bitcast(state, I32X4, builder);
1776 state.push1(builder.ins().fcvt_from_uint(F32X4, a))
1777 }
1778 Operator::F64x2ConvertLowI32x4S => {
1779 let a = pop1_with_bitcast(state, I32X4, builder);
1780 state.push1(builder.ins().fcvt_low_from_sint(F64X2, a));
1781 }
1782 Operator::I32x4TruncSatF32x4S => {
1783 let a = pop1_with_bitcast(state, F32X4, builder);
1784 state.push1(builder.ins().fcvt_to_sint_sat(I32X4, a))
1785 }
1786 Operator::I32x4TruncSatF32x4U => {
1787 let a = pop1_with_bitcast(state, F32X4, builder);
1788 state.push1(builder.ins().fcvt_to_uint_sat(I32X4, a))
1789 }
1790 Operator::I8x16NarrowI16x8S => {
1791 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
1792 state.push1(builder.ins().snarrow(a, b))
1793 }
1794 Operator::I16x8NarrowI32x4S => {
1795 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
1796 state.push1(builder.ins().snarrow(a, b))
1797 }
1798 Operator::I8x16NarrowI16x8U => {
1799 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
1800 state.push1(builder.ins().unarrow(a, b))
1801 }
1802 Operator::I16x8NarrowI32x4U => {
1803 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
1804 state.push1(builder.ins().unarrow(a, b))
1805 }
1806 Operator::I16x8ExtendLowI8x16S => {
1807 let a = pop1_with_bitcast(state, I8X16, builder);
1808 state.push1(builder.ins().swiden_low(a))
1809 }
1810 Operator::I16x8ExtendHighI8x16S => {
1811 let a = pop1_with_bitcast(state, I8X16, builder);
1812 state.push1(builder.ins().swiden_high(a))
1813 }
1814 Operator::I16x8ExtendLowI8x16U => {
1815 let a = pop1_with_bitcast(state, I8X16, builder);
1816 state.push1(builder.ins().uwiden_low(a))
1817 }
1818 Operator::I16x8ExtendHighI8x16U => {
1819 let a = pop1_with_bitcast(state, I8X16, builder);
1820 state.push1(builder.ins().uwiden_high(a))
1821 }
1822 Operator::I32x4ExtendLowI16x8S => {
1823 let a = pop1_with_bitcast(state, I16X8, builder);
1824 state.push1(builder.ins().swiden_low(a))
1825 }
1826 Operator::I32x4ExtendHighI16x8S => {
1827 let a = pop1_with_bitcast(state, I16X8, builder);
1828 state.push1(builder.ins().swiden_high(a))
1829 }
1830 Operator::I32x4ExtendLowI16x8U => {
1831 let a = pop1_with_bitcast(state, I16X8, builder);
1832 state.push1(builder.ins().uwiden_low(a))
1833 }
1834 Operator::I32x4ExtendHighI16x8U => {
1835 let a = pop1_with_bitcast(state, I16X8, builder);
1836 state.push1(builder.ins().uwiden_high(a))
1837 }
1838
1839 Operator::F32x4Ceil | Operator::F64x2Ceil => {
1840 // This is something of a misuse of `type_of`, because that produces the return type
1841 // of `op`. In this case we want the arg type, but we know it's the same as the
1842 // return type. Same for the 3 cases below.
1843 let arg = pop1_with_bitcast(state, type_of(op), builder);
1844 state.push1(builder.ins().ceil(arg));
1845 }
1846 Operator::F32x4Floor | Operator::F64x2Floor => {
1847 let arg = pop1_with_bitcast(state, type_of(op), builder);
1848 state.push1(builder.ins().floor(arg));
1849 }
1850 Operator::F32x4Trunc | Operator::F64x2Trunc => {
1851 let arg = pop1_with_bitcast(state, type_of(op), builder);
1852 state.push1(builder.ins().trunc(arg));
1853 }
1854 Operator::F32x4Nearest | Operator::F64x2Nearest => {
1855 let arg = pop1_with_bitcast(state, type_of(op), builder);
1856 state.push1(builder.ins().nearest(arg));
1857 }
1858 Operator::I32x4DotI16x8S => {
1859 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
1860 state.push1(builder.ins().widening_pairwise_dot_product_s(a, b));
1861 }
1862 Operator::I64x2ExtendLowI32x4S
1863 | Operator::I64x2ExtendHighI32x4S
1864 | Operator::I64x2ExtendLowI32x4U
1865 | Operator::I64x2ExtendHighI32x4U
1866 | Operator::I16x8Q15MulrSatS
1867 | Operator::I16x8ExtMulLowI8x16S
1868 | Operator::I16x8ExtMulHighI8x16S
1869 | Operator::I16x8ExtMulLowI8x16U
1870 | Operator::I16x8ExtMulHighI8x16U
1871 | Operator::I32x4ExtMulLowI16x8S
1872 | Operator::I32x4ExtMulHighI16x8S
1873 | Operator::I32x4ExtMulLowI16x8U
1874 | Operator::I32x4ExtMulHighI16x8U
1875 | Operator::I64x2ExtMulLowI32x4S
1876 | Operator::I64x2ExtMulHighI32x4S
1877 | Operator::I64x2ExtMulLowI32x4U
1878 | Operator::I64x2ExtMulHighI32x4U
1879 | Operator::I16x8ExtAddPairwiseI8x16S
1880 | Operator::I16x8ExtAddPairwiseI8x16U
1881 | Operator::I32x4ExtAddPairwiseI16x8S
1882 | Operator::I32x4ExtAddPairwiseI16x8U
1883 | Operator::F32x4DemoteF64x2Zero
1884 | Operator::F64x2PromoteLowF32x4
1885 | Operator::F64x2ConvertLowI32x4U
1886 | Operator::I32x4TruncSatF64x2SZero
1887 | Operator::I32x4TruncSatF64x2UZero
1888 | Operator::I8x16Popcnt => {
1889 return Err(wasm_unsupported!("proposed simd operator {:?}", op));
1890 }
1891 Operator::ReturnCall { .. } | Operator::ReturnCallIndirect { .. } => {
1892 return Err(wasm_unsupported!("proposed tail-call operator {:?}", op));
1893 }
1894 };
1895 Ok(())
1896 }
1897
1898 // Clippy warns us of some fields we are deliberately ignoring
1899 #[cfg_attr(feature = "cargo-clippy", allow(clippy::unneeded_field_pattern))]
1900 /// Deals with a Wasm instruction located in an unreachable portion of the code. Most of them
1901 /// are dropped but special ones like `End` or `Else` signal the potential end of the unreachable
1902 /// portion so the translation state must be updated accordingly.
translate_unreachable_operator<FE: FuncEnvironment + ?Sized>( validator: &FuncValidator<impl WasmModuleResources>, op: &Operator, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>1903 fn translate_unreachable_operator<FE: FuncEnvironment + ?Sized>(
1904 validator: &FuncValidator<impl WasmModuleResources>,
1905 op: &Operator,
1906 builder: &mut FunctionBuilder,
1907 state: &mut FuncTranslationState,
1908 environ: &mut FE,
1909 ) -> WasmResult<()> {
1910 debug_assert!(!state.reachable);
1911 match *op {
1912 Operator::If { ty } => {
1913 // Push a placeholder control stack entry. The if isn't reachable,
1914 // so we don't have any branches anywhere.
1915 state.push_if(
1916 ir::Block::reserved_value(),
1917 ElseData::NoElse {
1918 branch_inst: ir::Inst::reserved_value(),
1919 },
1920 0,
1921 0,
1922 ty,
1923 );
1924 }
1925 Operator::Loop { ty: _ } | Operator::Block { ty: _ } => {
1926 state.push_block(ir::Block::reserved_value(), 0, 0);
1927 }
1928 Operator::Else => {
1929 let i = state.control_stack.len() - 1;
1930 match state.control_stack[i] {
1931 ControlStackFrame::If {
1932 ref else_data,
1933 head_is_reachable,
1934 ref mut consequent_ends_reachable,
1935 blocktype,
1936 ..
1937 } => {
1938 debug_assert!(consequent_ends_reachable.is_none());
1939 *consequent_ends_reachable = Some(state.reachable);
1940
1941 if head_is_reachable {
1942 // We have a branch from the head of the `if` to the `else`.
1943 state.reachable = true;
1944
1945 let else_block = match *else_data {
1946 ElseData::NoElse { branch_inst } => {
1947 let (params, _results) =
1948 blocktype_params_results(validator, blocktype)?;
1949 let else_block = block_with_params(builder, params, environ)?;
1950 let frame = state.control_stack.last().unwrap();
1951 frame.truncate_value_stack_to_else_params(&mut state.stack);
1952
1953 // We change the target of the branch instruction.
1954 builder.change_jump_destination(branch_inst, else_block);
1955 builder.seal_block(else_block);
1956 else_block
1957 }
1958 ElseData::WithElse { else_block } => {
1959 let frame = state.control_stack.last().unwrap();
1960 frame.truncate_value_stack_to_else_params(&mut state.stack);
1961 else_block
1962 }
1963 };
1964
1965 builder.switch_to_block(else_block);
1966
1967 // Again, no need to push the parameters for the `else`,
1968 // since we already did when we saw the original `if`. See
1969 // the comment for translating `Operator::Else` in
1970 // `translate_operator` for details.
1971 }
1972 }
1973 _ => unreachable!(),
1974 }
1975 }
1976 Operator::End => {
1977 let stack = &mut state.stack;
1978 let control_stack = &mut state.control_stack;
1979 let frame = control_stack.pop().unwrap();
1980
1981 // Pop unused parameters from stack.
1982 frame.truncate_value_stack_to_original_size(stack);
1983
1984 let reachable_anyway = match frame {
1985 // If it is a loop we also have to seal the body loop block
1986 ControlStackFrame::Loop { header, .. } => {
1987 builder.seal_block(header);
1988 // And loops can't have branches to the end.
1989 false
1990 }
1991 // If we never set `consequent_ends_reachable` then that means
1992 // we are finishing the consequent now, and there was no
1993 // `else`. Whether the following block is reachable depends only
1994 // on if the head was reachable.
1995 ControlStackFrame::If {
1996 head_is_reachable,
1997 consequent_ends_reachable: None,
1998 ..
1999 } => head_is_reachable,
2000 // Since we are only in this function when in unreachable code,
2001 // we know that the alternative just ended unreachable. Whether
2002 // the following block is reachable depends on if the consequent
2003 // ended reachable or not.
2004 ControlStackFrame::If {
2005 head_is_reachable,
2006 consequent_ends_reachable: Some(consequent_ends_reachable),
2007 ..
2008 } => head_is_reachable && consequent_ends_reachable,
2009 // All other control constructs are already handled.
2010 _ => false,
2011 };
2012
2013 if frame.exit_is_branched_to() || reachable_anyway {
2014 builder.switch_to_block(frame.following_code());
2015 builder.seal_block(frame.following_code());
2016
2017 // And add the return values of the block but only if the next block is reachable
2018 // (which corresponds to testing if the stack depth is 1)
2019 stack.extend_from_slice(builder.block_params(frame.following_code()));
2020 state.reachable = true;
2021 }
2022 }
2023 _ => {
2024 // We don't translate because this is unreachable code
2025 }
2026 }
2027
2028 Ok(())
2029 }
2030
2031 /// Get the address+offset to use for a heap access.
get_heap_addr( heap: ir::Heap, addr32: ir::Value, offset: u32, width: u32, addr_ty: Type, builder: &mut FunctionBuilder, ) -> (ir::Value, i32)2032 fn get_heap_addr(
2033 heap: ir::Heap,
2034 addr32: ir::Value,
2035 offset: u32,
2036 width: u32,
2037 addr_ty: Type,
2038 builder: &mut FunctionBuilder,
2039 ) -> (ir::Value, i32) {
2040 let offset_guard_size: u64 = builder.func.heaps[heap].offset_guard_size.into();
2041
2042 // How exactly the bounds check is performed here and what it's performed
2043 // on is a bit tricky. Generally we want to rely on access violations (e.g.
2044 // segfaults) to generate traps since that means we don't have to bounds
2045 // check anything explicitly.
2046 //
2047 // If we don't have a guard page of unmapped memory, though, then we can't
2048 // rely on this trapping behavior through segfaults. Instead we need to
2049 // bounds-check the entire memory access here which is everything from
2050 // `addr32 + offset` to `addr32 + offset + width` (not inclusive). In this
2051 // scenario our adjusted offset that we're checking is `offset + width`.
2052 //
2053 // If we have a guard page, however, then we can perform a further
2054 // optimization of the generated code by only checking multiples of the
2055 // offset-guard size to be more CSE-friendly. Knowing that we have at least
2056 // 1 page of a guard page we're then able to disregard the `width` since we
2057 // know it's always less than one page. Our bounds check will be for the
2058 // first byte which will either succeed and be guaranteed to fault if it's
2059 // actually out of bounds, or the bounds check itself will fail. In any case
2060 // we assert that the width is reasonably small for now so this assumption
2061 // can be adjusted in the future if we get larger widths.
2062 //
2063 // Put another way we can say, where `y < offset_guard_size`:
2064 //
2065 // n * offset_guard_size + y = offset
2066 //
2067 // We'll then pass `n * offset_guard_size` as the bounds check value. If
2068 // this traps then our `offset` would have trapped anyway. If this check
2069 // passes we know
2070 //
2071 // addr32 + n * offset_guard_size < bound
2072 //
2073 // which means
2074 //
2075 // addr32 + n * offset_guard_size + y < bound + offset_guard_size
2076 //
2077 // because `y < offset_guard_size`, which then means:
2078 //
2079 // addr32 + offset < bound + offset_guard_size
2080 //
2081 // Since we know that that guard size bytes are all unmapped we're
2082 // guaranteed that `offset` and the `width` bytes after it are either
2083 // in-bounds or will hit the guard page, meaning we'll get the desired
2084 // semantics we want.
2085 //
2086 // As one final comment on the bits with the guard size here, another goal
2087 // of this is to hit an optimization in `heap_addr` where if the heap size
2088 // minus the offset is >= 4GB then bounds checks are 100% eliminated. This
2089 // means that with huge guard regions (e.g. our 2GB default) most adjusted
2090 // offsets we're checking here are zero. This means that we'll hit the fast
2091 // path and emit zero conditional traps for bounds checks
2092 let adjusted_offset = if offset_guard_size == 0 {
2093 u64::from(offset) + u64::from(width)
2094 } else {
2095 assert!(width < 1024);
2096 cmp::max(u64::from(offset) / offset_guard_size * offset_guard_size, 1)
2097 };
2098 debug_assert!(adjusted_offset > 0); // want to bounds check at least 1 byte
2099 let check_size = u32::try_from(adjusted_offset).unwrap_or(u32::MAX);
2100 let base = builder.ins().heap_addr(addr_ty, heap, addr32, check_size);
2101
2102 // Native load/store instructions take a signed `Offset32` immediate, so adjust the base
2103 // pointer if necessary.
2104 if offset > i32::MAX as u32 {
2105 // Offset doesn't fit in the load/store instruction.
2106 let adj = builder.ins().iadd_imm(base, i64::from(i32::MAX) + 1);
2107 (adj, (offset - (i32::MAX as u32 + 1)) as i32)
2108 } else {
2109 (base, offset as i32)
2110 }
2111 }
2112
2113 /// Prepare for a load; factors out common functionality between load and load_extend operations.
prepare_load<FE: FuncEnvironment + ?Sized>( memarg: &MemoryImmediate, loaded_bytes: u32, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<(MemFlags, Value, Offset32)>2114 fn prepare_load<FE: FuncEnvironment + ?Sized>(
2115 memarg: &MemoryImmediate,
2116 loaded_bytes: u32,
2117 builder: &mut FunctionBuilder,
2118 state: &mut FuncTranslationState,
2119 environ: &mut FE,
2120 ) -> WasmResult<(MemFlags, Value, Offset32)> {
2121 let addr32 = state.pop1();
2122
2123 let heap = state.get_heap(builder.func, memarg.memory, environ)?;
2124 let (base, offset) = get_heap_addr(
2125 heap,
2126 addr32,
2127 memarg.offset,
2128 loaded_bytes,
2129 environ.pointer_type(),
2130 builder,
2131 );
2132
2133 // Note that we don't set `is_aligned` here, even if the load instruction's
2134 // alignment immediate says it's aligned, because WebAssembly's immediate
2135 // field is just a hint, while Cranelift's aligned flag needs a guarantee.
2136 // WebAssembly memory accesses are always little-endian.
2137 let mut flags = MemFlags::new();
2138 flags.set_endianness(ir::Endianness::Little);
2139
2140 Ok((flags, base, offset.into()))
2141 }
2142
2143 /// Translate a load instruction.
translate_load<FE: FuncEnvironment + ?Sized>( memarg: &MemoryImmediate, opcode: ir::Opcode, result_ty: Type, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2144 fn translate_load<FE: FuncEnvironment + ?Sized>(
2145 memarg: &MemoryImmediate,
2146 opcode: ir::Opcode,
2147 result_ty: Type,
2148 builder: &mut FunctionBuilder,
2149 state: &mut FuncTranslationState,
2150 environ: &mut FE,
2151 ) -> WasmResult<()> {
2152 let (flags, base, offset) = prepare_load(
2153 memarg,
2154 mem_op_size(opcode, result_ty),
2155 builder,
2156 state,
2157 environ,
2158 )?;
2159 let (load, dfg) = builder.ins().Load(opcode, result_ty, flags, offset, base);
2160 state.push1(dfg.first_result(load));
2161 Ok(())
2162 }
2163
2164 /// Translate a store instruction.
translate_store<FE: FuncEnvironment + ?Sized>( memarg: &MemoryImmediate, opcode: ir::Opcode, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2165 fn translate_store<FE: FuncEnvironment + ?Sized>(
2166 memarg: &MemoryImmediate,
2167 opcode: ir::Opcode,
2168 builder: &mut FunctionBuilder,
2169 state: &mut FuncTranslationState,
2170 environ: &mut FE,
2171 ) -> WasmResult<()> {
2172 let (addr32, val) = state.pop2();
2173 let val_ty = builder.func.dfg.value_type(val);
2174
2175 let heap = state.get_heap(builder.func, memarg.memory, environ)?;
2176 let (base, offset) = get_heap_addr(
2177 heap,
2178 addr32,
2179 memarg.offset,
2180 mem_op_size(opcode, val_ty),
2181 environ.pointer_type(),
2182 builder,
2183 );
2184 // See the comments in `prepare_load` about the flags.
2185 let mut flags = MemFlags::new();
2186 flags.set_endianness(ir::Endianness::Little);
2187 builder
2188 .ins()
2189 .Store(opcode, val_ty, flags, offset.into(), val, base);
2190 Ok(())
2191 }
2192
mem_op_size(opcode: ir::Opcode, ty: Type) -> u322193 fn mem_op_size(opcode: ir::Opcode, ty: Type) -> u32 {
2194 match opcode {
2195 ir::Opcode::Istore8 | ir::Opcode::Sload8 | ir::Opcode::Uload8 => 1,
2196 ir::Opcode::Istore16 | ir::Opcode::Sload16 | ir::Opcode::Uload16 => 2,
2197 ir::Opcode::Istore32 | ir::Opcode::Sload32 | ir::Opcode::Uload32 => 4,
2198 ir::Opcode::Store | ir::Opcode::Load => ty.bytes(),
2199 _ => panic!("unknown size of mem op for {:?}", opcode),
2200 }
2201 }
2202
translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState)2203 fn translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState) {
2204 let (arg0, arg1) = state.pop2();
2205 let val = builder.ins().icmp(cc, arg0, arg1);
2206 state.push1(builder.ins().bint(I32, val));
2207 }
2208
fold_atomic_mem_addr( linear_mem_addr: Value, memarg: &MemoryImmediate, access_ty: Type, builder: &mut FunctionBuilder, ) -> Value2209 fn fold_atomic_mem_addr(
2210 linear_mem_addr: Value,
2211 memarg: &MemoryImmediate,
2212 access_ty: Type,
2213 builder: &mut FunctionBuilder,
2214 ) -> Value {
2215 let access_ty_bytes = access_ty.bytes();
2216 let final_lma = if memarg.offset > 0 {
2217 assert!(builder.func.dfg.value_type(linear_mem_addr) == I32);
2218 let linear_mem_addr = builder.ins().uextend(I64, linear_mem_addr);
2219 let a = builder
2220 .ins()
2221 .iadd_imm(linear_mem_addr, i64::from(memarg.offset));
2222 let cflags = builder.ins().ifcmp_imm(a, 0x1_0000_0000i64);
2223 builder.ins().trapif(
2224 IntCC::UnsignedGreaterThanOrEqual,
2225 cflags,
2226 ir::TrapCode::HeapOutOfBounds,
2227 );
2228 builder.ins().ireduce(I32, a)
2229 } else {
2230 linear_mem_addr
2231 };
2232 assert!(access_ty_bytes == 4 || access_ty_bytes == 8);
2233 let final_lma_misalignment = builder
2234 .ins()
2235 .band_imm(final_lma, i64::from(access_ty_bytes - 1));
2236 let f = builder
2237 .ins()
2238 .ifcmp_imm(final_lma_misalignment, i64::from(0));
2239 builder
2240 .ins()
2241 .trapif(IntCC::NotEqual, f, ir::TrapCode::HeapMisaligned);
2242 final_lma
2243 }
2244
2245 // For an atomic memory operation, emit an alignment check for the linear memory address,
2246 // and then compute the final effective address.
finalise_atomic_mem_addr<FE: FuncEnvironment + ?Sized>( linear_mem_addr: Value, memarg: &MemoryImmediate, access_ty: Type, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<Value>2247 fn finalise_atomic_mem_addr<FE: FuncEnvironment + ?Sized>(
2248 linear_mem_addr: Value,
2249 memarg: &MemoryImmediate,
2250 access_ty: Type,
2251 builder: &mut FunctionBuilder,
2252 state: &mut FuncTranslationState,
2253 environ: &mut FE,
2254 ) -> WasmResult<Value> {
2255 // Check the alignment of `linear_mem_addr`.
2256 let access_ty_bytes = access_ty.bytes();
2257 let final_lma = builder
2258 .ins()
2259 .iadd_imm(linear_mem_addr, i64::from(memarg.offset));
2260 if access_ty_bytes != 1 {
2261 assert!(access_ty_bytes == 2 || access_ty_bytes == 4 || access_ty_bytes == 8);
2262 let final_lma_misalignment = builder
2263 .ins()
2264 .band_imm(final_lma, i64::from(access_ty_bytes - 1));
2265 let f = builder
2266 .ins()
2267 .ifcmp_imm(final_lma_misalignment, i64::from(0));
2268 builder
2269 .ins()
2270 .trapif(IntCC::NotEqual, f, ir::TrapCode::HeapMisaligned);
2271 }
2272
2273 // Compute the final effective address.
2274 let heap = state.get_heap(builder.func, memarg.memory, environ)?;
2275 let (base, offset) = get_heap_addr(
2276 heap,
2277 final_lma,
2278 /*offset=*/ 0,
2279 access_ty.bytes(),
2280 environ.pointer_type(),
2281 builder,
2282 );
2283
2284 let final_effective_address = builder.ins().iadd_imm(base, i64::from(offset));
2285 Ok(final_effective_address)
2286 }
2287
translate_atomic_rmw<FE: FuncEnvironment + ?Sized>( widened_ty: Type, access_ty: Type, op: AtomicRmwOp, memarg: &MemoryImmediate, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2288 fn translate_atomic_rmw<FE: FuncEnvironment + ?Sized>(
2289 widened_ty: Type,
2290 access_ty: Type,
2291 op: AtomicRmwOp,
2292 memarg: &MemoryImmediate,
2293 builder: &mut FunctionBuilder,
2294 state: &mut FuncTranslationState,
2295 environ: &mut FE,
2296 ) -> WasmResult<()> {
2297 let (linear_mem_addr, mut arg2) = state.pop2();
2298 let arg2_ty = builder.func.dfg.value_type(arg2);
2299
2300 // The operation is performed at type `access_ty`, and the old value is zero-extended
2301 // to type `widened_ty`.
2302 match access_ty {
2303 I8 | I16 | I32 | I64 => {}
2304 _ => {
2305 return Err(wasm_unsupported!(
2306 "atomic_rmw: unsupported access type {:?}",
2307 access_ty
2308 ))
2309 }
2310 };
2311 let w_ty_ok = match widened_ty {
2312 I32 | I64 => true,
2313 _ => false,
2314 };
2315 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
2316
2317 assert!(arg2_ty.bytes() >= access_ty.bytes());
2318 if arg2_ty.bytes() > access_ty.bytes() {
2319 arg2 = builder.ins().ireduce(access_ty, arg2);
2320 }
2321
2322 let final_effective_address =
2323 finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
2324
2325 // See the comments in `prepare_load` about the flags.
2326 let mut flags = MemFlags::new();
2327 flags.set_endianness(ir::Endianness::Little);
2328 let mut res = builder
2329 .ins()
2330 .atomic_rmw(access_ty, flags, op, final_effective_address, arg2);
2331 if access_ty != widened_ty {
2332 res = builder.ins().uextend(widened_ty, res);
2333 }
2334 state.push1(res);
2335 Ok(())
2336 }
2337
translate_atomic_cas<FE: FuncEnvironment + ?Sized>( widened_ty: Type, access_ty: Type, memarg: &MemoryImmediate, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2338 fn translate_atomic_cas<FE: FuncEnvironment + ?Sized>(
2339 widened_ty: Type,
2340 access_ty: Type,
2341 memarg: &MemoryImmediate,
2342 builder: &mut FunctionBuilder,
2343 state: &mut FuncTranslationState,
2344 environ: &mut FE,
2345 ) -> WasmResult<()> {
2346 let (linear_mem_addr, mut expected, mut replacement) = state.pop3();
2347 let expected_ty = builder.func.dfg.value_type(expected);
2348 let replacement_ty = builder.func.dfg.value_type(replacement);
2349
2350 // The compare-and-swap is performed at type `access_ty`, and the old value is zero-extended
2351 // to type `widened_ty`.
2352 match access_ty {
2353 I8 | I16 | I32 | I64 => {}
2354 _ => {
2355 return Err(wasm_unsupported!(
2356 "atomic_cas: unsupported access type {:?}",
2357 access_ty
2358 ))
2359 }
2360 };
2361 let w_ty_ok = match widened_ty {
2362 I32 | I64 => true,
2363 _ => false,
2364 };
2365 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
2366
2367 assert!(expected_ty.bytes() >= access_ty.bytes());
2368 if expected_ty.bytes() > access_ty.bytes() {
2369 expected = builder.ins().ireduce(access_ty, expected);
2370 }
2371 assert!(replacement_ty.bytes() >= access_ty.bytes());
2372 if replacement_ty.bytes() > access_ty.bytes() {
2373 replacement = builder.ins().ireduce(access_ty, replacement);
2374 }
2375
2376 let final_effective_address =
2377 finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
2378
2379 // See the comments in `prepare_load` about the flags.
2380 let mut flags = MemFlags::new();
2381 flags.set_endianness(ir::Endianness::Little);
2382 let mut res = builder
2383 .ins()
2384 .atomic_cas(flags, final_effective_address, expected, replacement);
2385 if access_ty != widened_ty {
2386 res = builder.ins().uextend(widened_ty, res);
2387 }
2388 state.push1(res);
2389 Ok(())
2390 }
2391
translate_atomic_load<FE: FuncEnvironment + ?Sized>( widened_ty: Type, access_ty: Type, memarg: &MemoryImmediate, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2392 fn translate_atomic_load<FE: FuncEnvironment + ?Sized>(
2393 widened_ty: Type,
2394 access_ty: Type,
2395 memarg: &MemoryImmediate,
2396 builder: &mut FunctionBuilder,
2397 state: &mut FuncTranslationState,
2398 environ: &mut FE,
2399 ) -> WasmResult<()> {
2400 let linear_mem_addr = state.pop1();
2401
2402 // The load is performed at type `access_ty`, and the loaded value is zero extended
2403 // to `widened_ty`.
2404 match access_ty {
2405 I8 | I16 | I32 | I64 => {}
2406 _ => {
2407 return Err(wasm_unsupported!(
2408 "atomic_load: unsupported access type {:?}",
2409 access_ty
2410 ))
2411 }
2412 };
2413 let w_ty_ok = match widened_ty {
2414 I32 | I64 => true,
2415 _ => false,
2416 };
2417 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
2418
2419 let final_effective_address =
2420 finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
2421
2422 // See the comments in `prepare_load` about the flags.
2423 let mut flags = MemFlags::new();
2424 flags.set_endianness(ir::Endianness::Little);
2425 let mut res = builder
2426 .ins()
2427 .atomic_load(access_ty, flags, final_effective_address);
2428 if access_ty != widened_ty {
2429 res = builder.ins().uextend(widened_ty, res);
2430 }
2431 state.push1(res);
2432 Ok(())
2433 }
2434
translate_atomic_store<FE: FuncEnvironment + ?Sized>( access_ty: Type, memarg: &MemoryImmediate, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, environ: &mut FE, ) -> WasmResult<()>2435 fn translate_atomic_store<FE: FuncEnvironment + ?Sized>(
2436 access_ty: Type,
2437 memarg: &MemoryImmediate,
2438 builder: &mut FunctionBuilder,
2439 state: &mut FuncTranslationState,
2440 environ: &mut FE,
2441 ) -> WasmResult<()> {
2442 let (linear_mem_addr, mut data) = state.pop2();
2443 let data_ty = builder.func.dfg.value_type(data);
2444
2445 // The operation is performed at type `access_ty`, and the data to be stored may first
2446 // need to be narrowed accordingly.
2447 match access_ty {
2448 I8 | I16 | I32 | I64 => {}
2449 _ => {
2450 return Err(wasm_unsupported!(
2451 "atomic_store: unsupported access type {:?}",
2452 access_ty
2453 ))
2454 }
2455 };
2456 let d_ty_ok = match data_ty {
2457 I32 | I64 => true,
2458 _ => false,
2459 };
2460 assert!(d_ty_ok && data_ty.bytes() >= access_ty.bytes());
2461
2462 if data_ty.bytes() > access_ty.bytes() {
2463 data = builder.ins().ireduce(access_ty, data);
2464 }
2465
2466 let final_effective_address =
2467 finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
2468
2469 // See the comments in `prepare_load` about the flags.
2470 let mut flags = MemFlags::new();
2471 flags.set_endianness(ir::Endianness::Little);
2472 builder
2473 .ins()
2474 .atomic_store(flags, data, final_effective_address);
2475 Ok(())
2476 }
2477
translate_vector_icmp( cc: IntCC, needed_type: Type, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, )2478 fn translate_vector_icmp(
2479 cc: IntCC,
2480 needed_type: Type,
2481 builder: &mut FunctionBuilder,
2482 state: &mut FuncTranslationState,
2483 ) {
2484 let (a, b) = state.pop2();
2485 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
2486 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
2487 state.push1(builder.ins().icmp(cc, bitcast_a, bitcast_b))
2488 }
2489
translate_fcmp(cc: FloatCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState)2490 fn translate_fcmp(cc: FloatCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState) {
2491 let (arg0, arg1) = state.pop2();
2492 let val = builder.ins().fcmp(cc, arg0, arg1);
2493 state.push1(builder.ins().bint(I32, val));
2494 }
2495
translate_vector_fcmp( cc: FloatCC, needed_type: Type, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, )2496 fn translate_vector_fcmp(
2497 cc: FloatCC,
2498 needed_type: Type,
2499 builder: &mut FunctionBuilder,
2500 state: &mut FuncTranslationState,
2501 ) {
2502 let (a, b) = state.pop2();
2503 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
2504 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
2505 state.push1(builder.ins().fcmp(cc, bitcast_a, bitcast_b))
2506 }
2507
translate_br_if( relative_depth: u32, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, )2508 fn translate_br_if(
2509 relative_depth: u32,
2510 builder: &mut FunctionBuilder,
2511 state: &mut FuncTranslationState,
2512 ) {
2513 let val = state.pop1();
2514 let (br_destination, inputs) = translate_br_if_args(relative_depth, state);
2515 canonicalise_then_brnz(builder, val, br_destination, inputs);
2516
2517 let next_block = builder.create_block();
2518 canonicalise_then_jump(builder, next_block, &[]);
2519 builder.seal_block(next_block); // The only predecessor is the current block.
2520 builder.switch_to_block(next_block);
2521 }
2522
translate_br_if_args( relative_depth: u32, state: &mut FuncTranslationState, ) -> (ir::Block, &mut [ir::Value])2523 fn translate_br_if_args(
2524 relative_depth: u32,
2525 state: &mut FuncTranslationState,
2526 ) -> (ir::Block, &mut [ir::Value]) {
2527 let i = state.control_stack.len() - 1 - (relative_depth as usize);
2528 let (return_count, br_destination) = {
2529 let frame = &mut state.control_stack[i];
2530 // The values returned by the branch are still available for the reachable
2531 // code that comes after it
2532 frame.set_branched_to_exit();
2533 let return_count = if frame.is_loop() {
2534 frame.num_param_values()
2535 } else {
2536 frame.num_return_values()
2537 };
2538 (return_count, frame.br_destination())
2539 };
2540 let inputs = state.peekn_mut(return_count);
2541 (br_destination, inputs)
2542 }
2543
2544 /// Determine the returned value type of a WebAssembly operator
type_of(operator: &Operator) -> Type2545 fn type_of(operator: &Operator) -> Type {
2546 match operator {
2547 Operator::V128Load { .. }
2548 | Operator::V128Store { .. }
2549 | Operator::V128Const { .. }
2550 | Operator::V128Not
2551 | Operator::V128And
2552 | Operator::V128AndNot
2553 | Operator::V128Or
2554 | Operator::V128Xor
2555 | Operator::V128AnyTrue
2556 | Operator::V128Bitselect => I8X16, // default type representing V128
2557
2558 Operator::I8x16Shuffle { .. }
2559 | Operator::I8x16Splat
2560 | Operator::V128Load8Splat { .. }
2561 | Operator::V128Load8Lane { .. }
2562 | Operator::V128Store8Lane { .. }
2563 | Operator::I8x16ExtractLaneS { .. }
2564 | Operator::I8x16ExtractLaneU { .. }
2565 | Operator::I8x16ReplaceLane { .. }
2566 | Operator::I8x16Eq
2567 | Operator::I8x16Ne
2568 | Operator::I8x16LtS
2569 | Operator::I8x16LtU
2570 | Operator::I8x16GtS
2571 | Operator::I8x16GtU
2572 | Operator::I8x16LeS
2573 | Operator::I8x16LeU
2574 | Operator::I8x16GeS
2575 | Operator::I8x16GeU
2576 | Operator::I8x16Neg
2577 | Operator::I8x16Abs
2578 | Operator::I8x16AllTrue
2579 | Operator::I8x16Shl
2580 | Operator::I8x16ShrS
2581 | Operator::I8x16ShrU
2582 | Operator::I8x16Add
2583 | Operator::I8x16AddSatS
2584 | Operator::I8x16AddSatU
2585 | Operator::I8x16Sub
2586 | Operator::I8x16SubSatS
2587 | Operator::I8x16SubSatU
2588 | Operator::I8x16MinS
2589 | Operator::I8x16MinU
2590 | Operator::I8x16MaxS
2591 | Operator::I8x16MaxU
2592 | Operator::I8x16RoundingAverageU
2593 | Operator::I8x16Bitmask => I8X16,
2594
2595 Operator::I16x8Splat
2596 | Operator::V128Load16Splat { .. }
2597 | Operator::V128Load16Lane { .. }
2598 | Operator::V128Store16Lane { .. }
2599 | Operator::I16x8ExtractLaneS { .. }
2600 | Operator::I16x8ExtractLaneU { .. }
2601 | Operator::I16x8ReplaceLane { .. }
2602 | Operator::I16x8Eq
2603 | Operator::I16x8Ne
2604 | Operator::I16x8LtS
2605 | Operator::I16x8LtU
2606 | Operator::I16x8GtS
2607 | Operator::I16x8GtU
2608 | Operator::I16x8LeS
2609 | Operator::I16x8LeU
2610 | Operator::I16x8GeS
2611 | Operator::I16x8GeU
2612 | Operator::I16x8Neg
2613 | Operator::I16x8Abs
2614 | Operator::I16x8AllTrue
2615 | Operator::I16x8Shl
2616 | Operator::I16x8ShrS
2617 | Operator::I16x8ShrU
2618 | Operator::I16x8Add
2619 | Operator::I16x8AddSatS
2620 | Operator::I16x8AddSatU
2621 | Operator::I16x8Sub
2622 | Operator::I16x8SubSatS
2623 | Operator::I16x8SubSatU
2624 | Operator::I16x8MinS
2625 | Operator::I16x8MinU
2626 | Operator::I16x8MaxS
2627 | Operator::I16x8MaxU
2628 | Operator::I16x8RoundingAverageU
2629 | Operator::I16x8Mul
2630 | Operator::I16x8Bitmask => I16X8,
2631
2632 Operator::I32x4Splat
2633 | Operator::V128Load32Splat { .. }
2634 | Operator::V128Load32Lane { .. }
2635 | Operator::V128Store32Lane { .. }
2636 | Operator::I32x4ExtractLane { .. }
2637 | Operator::I32x4ReplaceLane { .. }
2638 | Operator::I32x4Eq
2639 | Operator::I32x4Ne
2640 | Operator::I32x4LtS
2641 | Operator::I32x4LtU
2642 | Operator::I32x4GtS
2643 | Operator::I32x4GtU
2644 | Operator::I32x4LeS
2645 | Operator::I32x4LeU
2646 | Operator::I32x4GeS
2647 | Operator::I32x4GeU
2648 | Operator::I32x4Neg
2649 | Operator::I32x4Abs
2650 | Operator::I32x4AllTrue
2651 | Operator::I32x4Shl
2652 | Operator::I32x4ShrS
2653 | Operator::I32x4ShrU
2654 | Operator::I32x4Add
2655 | Operator::I32x4Sub
2656 | Operator::I32x4Mul
2657 | Operator::I32x4MinS
2658 | Operator::I32x4MinU
2659 | Operator::I32x4MaxS
2660 | Operator::I32x4MaxU
2661 | Operator::F32x4ConvertI32x4S
2662 | Operator::F32x4ConvertI32x4U
2663 | Operator::I32x4Bitmask
2664 | Operator::V128Load32Zero { .. } => I32X4,
2665
2666 Operator::I64x2Splat
2667 | Operator::V128Load64Splat { .. }
2668 | Operator::V128Load64Lane { .. }
2669 | Operator::V128Store64Lane { .. }
2670 | Operator::I64x2ExtractLane { .. }
2671 | Operator::I64x2ReplaceLane { .. }
2672 | Operator::I64x2Eq
2673 | Operator::I64x2Ne
2674 | Operator::I64x2LtS
2675 | Operator::I64x2GtS
2676 | Operator::I64x2LeS
2677 | Operator::I64x2GeS
2678 | Operator::I64x2Neg
2679 | Operator::I64x2Abs
2680 | Operator::I64x2AllTrue
2681 | Operator::I64x2Shl
2682 | Operator::I64x2ShrS
2683 | Operator::I64x2ShrU
2684 | Operator::I64x2Add
2685 | Operator::I64x2Sub
2686 | Operator::I64x2Mul
2687 | Operator::I64x2Bitmask
2688 | Operator::V128Load64Zero { .. } => I64X2,
2689
2690 Operator::F32x4Splat
2691 | Operator::F32x4ExtractLane { .. }
2692 | Operator::F32x4ReplaceLane { .. }
2693 | Operator::F32x4Eq
2694 | Operator::F32x4Ne
2695 | Operator::F32x4Lt
2696 | Operator::F32x4Gt
2697 | Operator::F32x4Le
2698 | Operator::F32x4Ge
2699 | Operator::F32x4Abs
2700 | Operator::F32x4Neg
2701 | Operator::F32x4Sqrt
2702 | Operator::F32x4Add
2703 | Operator::F32x4Sub
2704 | Operator::F32x4Mul
2705 | Operator::F32x4Div
2706 | Operator::F32x4Min
2707 | Operator::F32x4Max
2708 | Operator::F32x4PMin
2709 | Operator::F32x4PMax
2710 | Operator::I32x4TruncSatF32x4S
2711 | Operator::I32x4TruncSatF32x4U
2712 | Operator::F32x4Ceil
2713 | Operator::F32x4Floor
2714 | Operator::F32x4Trunc
2715 | Operator::F32x4Nearest => F32X4,
2716
2717 Operator::F64x2Splat
2718 | Operator::F64x2ExtractLane { .. }
2719 | Operator::F64x2ReplaceLane { .. }
2720 | Operator::F64x2Eq
2721 | Operator::F64x2Ne
2722 | Operator::F64x2Lt
2723 | Operator::F64x2Gt
2724 | Operator::F64x2Le
2725 | Operator::F64x2Ge
2726 | Operator::F64x2Abs
2727 | Operator::F64x2Neg
2728 | Operator::F64x2Sqrt
2729 | Operator::F64x2Add
2730 | Operator::F64x2Sub
2731 | Operator::F64x2Mul
2732 | Operator::F64x2Div
2733 | Operator::F64x2Min
2734 | Operator::F64x2Max
2735 | Operator::F64x2PMin
2736 | Operator::F64x2PMax
2737 | Operator::F64x2Ceil
2738 | Operator::F64x2Floor
2739 | Operator::F64x2Trunc
2740 | Operator::F64x2Nearest => F64X2,
2741
2742 _ => unimplemented!(
2743 "Currently only SIMD instructions are mapped to their return type; the \
2744 following instruction is not mapped: {:?}",
2745 operator
2746 ),
2747 }
2748 }
2749
2750 /// Some SIMD operations only operate on I8X16 in CLIF; this will convert them to that type by
2751 /// adding a raw_bitcast if necessary.
optionally_bitcast_vector( value: Value, needed_type: Type, builder: &mut FunctionBuilder, ) -> Value2752 fn optionally_bitcast_vector(
2753 value: Value,
2754 needed_type: Type,
2755 builder: &mut FunctionBuilder,
2756 ) -> Value {
2757 if builder.func.dfg.value_type(value) != needed_type {
2758 builder.ins().raw_bitcast(needed_type, value)
2759 } else {
2760 value
2761 }
2762 }
2763
2764 #[inline(always)]
is_non_canonical_v128(ty: ir::Type) -> bool2765 fn is_non_canonical_v128(ty: ir::Type) -> bool {
2766 match ty {
2767 B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2 => true,
2768 _ => false,
2769 }
2770 }
2771
2772 /// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not
2773 /// I8X16), and return them in a slice. A pre-scan is made to determine whether any casts are
2774 /// actually necessary, and if not, the original slice is returned. Otherwise the cast values
2775 /// are returned in a slice that belongs to the caller-supplied `SmallVec`.
canonicalise_v128_values<'a>( tmp_canonicalised: &'a mut SmallVec<[ir::Value; 16]>, builder: &mut FunctionBuilder, values: &'a [ir::Value], ) -> &'a [ir::Value]2776 fn canonicalise_v128_values<'a>(
2777 tmp_canonicalised: &'a mut SmallVec<[ir::Value; 16]>,
2778 builder: &mut FunctionBuilder,
2779 values: &'a [ir::Value],
2780 ) -> &'a [ir::Value] {
2781 debug_assert!(tmp_canonicalised.is_empty());
2782 // First figure out if any of the parameters need to be cast. Mostly they don't need to be.
2783 let any_non_canonical = values
2784 .iter()
2785 .any(|v| is_non_canonical_v128(builder.func.dfg.value_type(*v)));
2786 // Hopefully we take this exit most of the time, hence doing no heap allocation.
2787 if !any_non_canonical {
2788 return values;
2789 }
2790 // Otherwise we'll have to cast, and push the resulting `Value`s into `canonicalised`.
2791 for v in values {
2792 tmp_canonicalised.push(if is_non_canonical_v128(builder.func.dfg.value_type(*v)) {
2793 builder.ins().raw_bitcast(I8X16, *v)
2794 } else {
2795 *v
2796 });
2797 }
2798 tmp_canonicalised.as_slice()
2799 }
2800
2801 /// Generate a `jump` instruction, but first cast all 128-bit vector values to I8X16 if they
2802 /// don't have that type. This is done in somewhat roundabout way so as to ensure that we
2803 /// almost never have to do any heap allocation.
canonicalise_then_jump( builder: &mut FunctionBuilder, destination: ir::Block, params: &[ir::Value], ) -> ir::Inst2804 fn canonicalise_then_jump(
2805 builder: &mut FunctionBuilder,
2806 destination: ir::Block,
2807 params: &[ir::Value],
2808 ) -> ir::Inst {
2809 let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new();
2810 let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params);
2811 builder.ins().jump(destination, canonicalised)
2812 }
2813
2814 /// The same but for a `brz` instruction.
canonicalise_then_brz( builder: &mut FunctionBuilder, cond: ir::Value, destination: ir::Block, params: &[Value], ) -> ir::Inst2815 fn canonicalise_then_brz(
2816 builder: &mut FunctionBuilder,
2817 cond: ir::Value,
2818 destination: ir::Block,
2819 params: &[Value],
2820 ) -> ir::Inst {
2821 let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new();
2822 let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params);
2823 builder.ins().brz(cond, destination, canonicalised)
2824 }
2825
2826 /// The same but for a `brnz` instruction.
canonicalise_then_brnz( builder: &mut FunctionBuilder, cond: ir::Value, destination: ir::Block, params: &[Value], ) -> ir::Inst2827 fn canonicalise_then_brnz(
2828 builder: &mut FunctionBuilder,
2829 cond: ir::Value,
2830 destination: ir::Block,
2831 params: &[Value],
2832 ) -> ir::Inst {
2833 let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new();
2834 let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params);
2835 builder.ins().brnz(cond, destination, canonicalised)
2836 }
2837
2838 /// A helper for popping and bitcasting a single value; since SIMD values can lose their type by
2839 /// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
2840 /// typing issues.
pop1_with_bitcast( state: &mut FuncTranslationState, needed_type: Type, builder: &mut FunctionBuilder, ) -> Value2841 fn pop1_with_bitcast(
2842 state: &mut FuncTranslationState,
2843 needed_type: Type,
2844 builder: &mut FunctionBuilder,
2845 ) -> Value {
2846 optionally_bitcast_vector(state.pop1(), needed_type, builder)
2847 }
2848
2849 /// A helper for popping and bitcasting two values; since SIMD values can lose their type by
2850 /// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
2851 /// typing issues.
pop2_with_bitcast( state: &mut FuncTranslationState, needed_type: Type, builder: &mut FunctionBuilder, ) -> (Value, Value)2852 fn pop2_with_bitcast(
2853 state: &mut FuncTranslationState,
2854 needed_type: Type,
2855 builder: &mut FunctionBuilder,
2856 ) -> (Value, Value) {
2857 let (a, b) = state.pop2();
2858 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
2859 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
2860 (bitcast_a, bitcast_b)
2861 }
2862
2863 /// A helper for bitcasting a sequence of values (e.g. function arguments). If a value is a
2864 /// vector type that does not match its expected type, this will modify the value in place to point
2865 /// to the result of a `raw_bitcast`. This conversion is necessary to translate Wasm code that
2866 /// uses `V128` as function parameters (or implicitly in block parameters) and still use specific
2867 /// CLIF types (e.g. `I32X4`) in the function body.
bitcast_arguments( arguments: &mut [Value], expected_types: &[Type], builder: &mut FunctionBuilder, )2868 pub fn bitcast_arguments(
2869 arguments: &mut [Value],
2870 expected_types: &[Type],
2871 builder: &mut FunctionBuilder,
2872 ) {
2873 assert_eq!(arguments.len(), expected_types.len());
2874 for (i, t) in expected_types.iter().enumerate() {
2875 if t.is_vector() {
2876 assert!(
2877 builder.func.dfg.value_type(arguments[i]).is_vector(),
2878 "unexpected type mismatch: expected {}, argument {} was actually of type {}",
2879 t,
2880 arguments[i],
2881 builder.func.dfg.value_type(arguments[i])
2882 );
2883 arguments[i] = optionally_bitcast_vector(arguments[i], *t, builder)
2884 }
2885 }
2886 }
2887
2888 /// A helper to extract all the `Type` listings of each variable in `params`
2889 /// for only parameters the return true for `is_wasm`, typically paired with
2890 /// `is_wasm_return` or `is_wasm_parameter`.
wasm_param_types(params: &[ir::AbiParam], is_wasm: impl Fn(usize) -> bool) -> Vec<Type>2891 pub fn wasm_param_types(params: &[ir::AbiParam], is_wasm: impl Fn(usize) -> bool) -> Vec<Type> {
2892 let mut ret = Vec::with_capacity(params.len());
2893 for (i, param) in params.iter().enumerate() {
2894 if is_wasm(i) {
2895 ret.push(param.value_type);
2896 }
2897 }
2898 ret
2899 }
2900