1 //! This module implements lowering (instruction selection) from Cranelift IR
2 //! to machine instructions with virtual registers. This is *almost* the final
3 //! machine code, except for register allocation.
4
5 // TODO: separate the IR-query core of `LowerCtx` from the lowering logic built
6 // on top of it, e.g. the side-effect/coloring analysis and the scan support.
7
8 use crate::data_value::DataValue;
9 use crate::entity::SecondaryMap;
10 use crate::fx::{FxHashMap, FxHashSet};
11 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
12 use crate::ir::instructions::BranchInfo;
13 use crate::ir::{
14 ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
15 InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
16 ValueLabelAssignments, ValueLabelStart,
17 };
18 use crate::machinst::{
19 writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
20 VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs,
21 };
22 use crate::CodegenResult;
23 use alloc::boxed::Box;
24 use alloc::vec::Vec;
25 use core::convert::TryInto;
26 use log::debug;
27 use regalloc::{Reg, StackmapRequestInfo, Writable};
28 use smallvec::{smallvec, SmallVec};
29 use std::fmt::Debug;
30
31 /// An "instruction color" partitions CLIF instructions by side-effecting ops.
32 /// All instructions with the same "color" are guaranteed not to be separated by
33 /// any side-effecting op (for this purpose, loads are also considered
34 /// side-effecting, to avoid subtle questions w.r.t. the memory model), and
35 /// furthermore, it is guaranteed that for any two instructions A and B such
36 /// that color(A) == color(B), either A dominates B and B postdominates A, or
37 /// vice-versa. (For now, in practice, only ops in the same basic block can ever
38 /// have the same color, trivially providing the second condition.) Intuitively,
39 /// this means that the ops of the same color must always execute "together", as
40 /// part of one atomic contiguous section of the dynamic execution trace, and
41 /// they can be freely permuted (modulo true dataflow dependencies) without
42 /// affecting program behavior.
43 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
44 struct InstColor(u32);
45 impl InstColor {
new(n: u32) -> InstColor46 fn new(n: u32) -> InstColor {
47 InstColor(n)
48 }
49
50 /// Get an arbitrary index representing this color. The index is unique
51 /// *within a single function compilation*, but indices may be reused across
52 /// functions.
get(self) -> u3253 pub fn get(self) -> u32 {
54 self.0
55 }
56 }
57
58 /// A context that machine-specific lowering code can use to emit lowered
59 /// instructions. This is the view of the machine-independent per-function
60 /// lowering context that is seen by the machine backend.
61 pub trait LowerCtx {
62 /// The instruction type for which this lowering framework is instantiated.
63 type I: VCodeInst;
64
65 // Function-level queries:
66
67 /// Get the `ABICallee`.
abi(&mut self) -> &mut dyn ABICallee<I = Self::I>68 fn abi(&mut self) -> &mut dyn ABICallee<I = Self::I>;
69 /// Get the (virtual) register that receives the return value. A return
70 /// instruction should lower into a sequence that fills this register. (Why
71 /// not allow the backend to specify its own result register for the return?
72 /// Because there may be multiple return points.)
retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>73 fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>;
74 /// Returns the vreg containing the VmContext parameter, if there's one.
get_vm_context(&self) -> Option<Reg>75 fn get_vm_context(&self) -> Option<Reg>;
76
77 // General instruction queries:
78
79 /// Get the instdata for a given IR instruction.
data(&self, ir_inst: Inst) -> &InstructionData80 fn data(&self, ir_inst: Inst) -> &InstructionData;
81 /// Get the controlling type for a polymorphic IR instruction.
ty(&self, ir_inst: Inst) -> Type82 fn ty(&self, ir_inst: Inst) -> Type;
83 /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple
84 /// providing this name and the "relocation distance", i.e., whether the backend
85 /// can assume the target will be "nearby" (within some small offset) or an
86 /// arbitrary address. (This comes from the `colocated` bit in the CLIF.)
call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>87 fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>;
88 /// Get the signature for a call or call-indirect instruction.
call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>89 fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>;
90 /// Get the symbol name, relocation distance estimate, and offset for a
91 /// symbol_value instruction.
symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>92 fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>;
93 /// Returns the memory flags of a given memory access.
memflags(&self, ir_inst: Inst) -> Option<MemFlags>94 fn memflags(&self, ir_inst: Inst) -> Option<MemFlags>;
95 /// Get the source location for a given instruction.
srcloc(&self, ir_inst: Inst) -> SourceLoc96 fn srcloc(&self, ir_inst: Inst) -> SourceLoc;
97
98 // Instruction input/output queries:
99
100 /// Get the number of inputs to the given IR instruction.
num_inputs(&self, ir_inst: Inst) -> usize101 fn num_inputs(&self, ir_inst: Inst) -> usize;
102 /// Get the number of outputs to the given IR instruction.
num_outputs(&self, ir_inst: Inst) -> usize103 fn num_outputs(&self, ir_inst: Inst) -> usize;
104 /// Get the type for an instruction's input.
input_ty(&self, ir_inst: Inst, idx: usize) -> Type105 fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type;
106 /// Get the type for an instruction's output.
output_ty(&self, ir_inst: Inst, idx: usize) -> Type107 fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type;
108 /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit
109 /// value, if possible.
get_constant(&self, ir_inst: Inst) -> Option<u64>110 fn get_constant(&self, ir_inst: Inst) -> Option<u64>;
111 /// Get the input as one of two options other than a direct register:
112 ///
113 /// - An instruction, given that it is effect-free or able to sink its
114 /// effect to the current instruction being lowered, and given it has only
115 /// one output, and if effect-ful, given that this is the only use;
116 /// - A constant, if the value is a constant.
117 ///
118 /// The instruction input may be available in either of these forms. It may
119 /// be available in neither form, if the conditions are not met; if so, use
120 /// `put_input_in_regs()` instead to get it in a register.
121 ///
122 /// If the backend merges the effect of a side-effecting instruction, it
123 /// must call `sink_inst()`. When this is called, it indicates that the
124 /// effect has been sunk to the current scan location. The sunk
125 /// instruction's result(s) must have *no* uses remaining, because it will
126 /// not be codegen'd (it has been integrated into the current instruction).
get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput127 fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput;
128 /// Put the `idx`th input into register(s) and return the assigned register.
put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>129 fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>;
130 /// Get the `idx`th output register(s) of the given IR instruction. When
131 /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that
132 /// the backend will write results to these output register(s). This
133 /// register will always be "fresh"; it is guaranteed not to overlap with
134 /// any of the inputs, and can be freely used as a scratch register within
135 /// the lowered instruction sequence, as long as its final value is the
136 /// result of the computation.
get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>137 fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>;
138
139 // Codegen primitives: allocate temps, emit instructions, set result registers,
140 // ask for an input to be gen'd into a register.
141
142 /// Get a new temp.
alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>143 fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>;
144 /// Emit a machine instruction.
emit(&mut self, mach_inst: Self::I)145 fn emit(&mut self, mach_inst: Self::I);
146 /// Emit a machine instruction that is a safepoint.
emit_safepoint(&mut self, mach_inst: Self::I)147 fn emit_safepoint(&mut self, mach_inst: Self::I);
148 /// Indicate that the side-effect of an instruction has been sunk to the
149 /// current scan location. This should only be done with the instruction's
150 /// original results are not used (i.e., `put_input_in_regs` is not invoked
151 /// for the input produced by the sunk instruction), otherwise the
152 /// side-effect will occur twice.
sink_inst(&mut self, ir_inst: Inst)153 fn sink_inst(&mut self, ir_inst: Inst);
154 /// Retrieve constant data given a handle.
get_constant_data(&self, constant_handle: Constant) -> &ConstantData155 fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData;
156 /// Indicate that a constant should be emitted.
use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant157 fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant;
158 /// Retrieve the value immediate from an instruction. This will perform necessary lookups on the
159 /// `DataFlowGraph` to retrieve even large immediates.
get_immediate(&self, ir_inst: Inst) -> Option<DataValue>160 fn get_immediate(&self, ir_inst: Inst) -> Option<DataValue>;
161 /// Cause the value in `reg` to be in a virtual reg, by copying it into a new virtual reg
162 /// if `reg` is a real reg. `ty` describes the type of the value in `reg`.
ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg163 fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg;
164 }
165
166 /// A representation of all of the ways in which a value is available, aside
167 /// from as a direct register.
168 ///
169 /// - An instruction, if it would be allowed to occur at the current location
170 /// instead (see [LowerCtx::get_input_as_source_or_const()] for more
171 /// details).
172 ///
173 /// - A constant, if the value is known to be a constant.
174 #[derive(Clone, Copy, Debug)]
175 pub struct NonRegInput {
176 /// An instruction produces this value (as the given output), and its
177 /// computation (and side-effect if applicable) could occur at the
178 /// current instruction's location instead.
179 ///
180 /// If this instruction's operation is merged into the current instruction,
181 /// the backend must call [LowerCtx::sink_inst()].
182 pub inst: Option<(Inst, usize)>,
183 /// The value is a known constant.
184 pub constant: Option<u64>,
185 }
186
187 /// A machine backend.
188 pub trait LowerBackend {
189 /// The machine instruction type.
190 type MInst: VCodeInst;
191
192 /// Lower a single instruction.
193 ///
194 /// For a branch, this function should not generate the actual branch
195 /// instruction. However, it must force any values it needs for the branch
196 /// edge (block-param actuals) into registers, because the actual branch
197 /// generation (`lower_branch_group()`) happens *after* any possible merged
198 /// out-edge.
lower<C: LowerCtx<I = Self::MInst>>(&self, ctx: &mut C, inst: Inst) -> CodegenResult<()>199 fn lower<C: LowerCtx<I = Self::MInst>>(&self, ctx: &mut C, inst: Inst) -> CodegenResult<()>;
200
201 /// Lower a block-terminating group of branches (which together can be seen
202 /// as one N-way branch), given a vcode MachLabel for each target.
lower_branch_group<C: LowerCtx<I = Self::MInst>>( &self, ctx: &mut C, insts: &[Inst], targets: &[MachLabel], ) -> CodegenResult<()>203 fn lower_branch_group<C: LowerCtx<I = Self::MInst>>(
204 &self,
205 ctx: &mut C,
206 insts: &[Inst],
207 targets: &[MachLabel],
208 ) -> CodegenResult<()>;
209
210 /// A bit of a hack: give a fixed register that always holds the result of a
211 /// `get_pinned_reg` instruction, if known. This allows elision of moves
212 /// into the associated vreg, instead using the real reg directly.
maybe_pinned_reg(&self) -> Option<Reg>213 fn maybe_pinned_reg(&self) -> Option<Reg> {
214 None
215 }
216 }
217
218 /// A pending instruction to insert and auxiliary information about it: its source location and
219 /// whether it is a safepoint.
220 struct InstTuple<I: VCodeInst> {
221 loc: SourceLoc,
222 is_safepoint: bool,
223 inst: I,
224 }
225
226 /// Machine-independent lowering driver / machine-instruction container. Maintains a correspondence
227 /// from original Inst to MachInsts.
228 pub struct Lower<'func, I: VCodeInst> {
229 /// The function to lower.
230 f: &'func Function,
231
232 /// Lowered machine instructions.
233 vcode: VCodeBuilder<I>,
234
235 /// Mapping from `Value` (SSA value in IR) to virtual register.
236 value_regs: SecondaryMap<Value, ValueRegs<Reg>>,
237
238 /// Return-value vregs.
239 retval_regs: Vec<ValueRegs<Reg>>,
240
241 /// Instruction colors at block exits. From this map, we can recover all
242 /// instruction colors by scanning backward from the block end and
243 /// decrementing on any color-changing (side-effecting) instruction.
244 block_end_colors: SecondaryMap<Block, InstColor>,
245
246 /// Instruction colors at side-effecting ops. This is the *entry* color,
247 /// i.e., the version of global state that exists before an instruction
248 /// executes. For each side-effecting instruction, the *exit* color is its
249 /// entry color plus one.
250 side_effect_inst_entry_colors: FxHashMap<Inst, InstColor>,
251
252 /// Current color as we scan during lowering. While we are lowering an
253 /// instruction, this is equal to the color *at entry to* the instruction.
254 cur_scan_entry_color: Option<InstColor>,
255
256 /// Current instruction as we scan during lowering.
257 cur_inst: Option<Inst>,
258
259 /// Instruction constant values, if known.
260 inst_constants: FxHashMap<Inst, u64>,
261
262 /// Use-counts per SSA value, as counted in the input IR.
263 value_uses: SecondaryMap<Value, u32>,
264
265 /// Actual uses of each SSA value so far, incremented while lowering.
266 value_lowered_uses: SecondaryMap<Value, u32>,
267
268 /// Effectful instructions that have been sunk; they are not codegen'd at
269 /// their original locations.
270 inst_sunk: FxHashSet<Inst>,
271
272 /// Next virtual register number to allocate.
273 next_vreg: u32,
274
275 /// Insts in reverse block order, before final copy to vcode.
276 block_insts: Vec<InstTuple<I>>,
277
278 /// Ranges in `block_insts` constituting BBs.
279 block_ranges: Vec<(usize, usize)>,
280
281 /// Instructions collected for the BB in progress, in reverse order, with
282 /// source-locs attached.
283 bb_insts: Vec<InstTuple<I>>,
284
285 /// Instructions collected for the CLIF inst in progress, in forward order.
286 ir_insts: Vec<InstTuple<I>>,
287
288 /// The register to use for GetPinnedReg, if any, on this architecture.
289 pinned_reg: Option<Reg>,
290
291 /// The vreg containing the special VmContext parameter, if it is present in the current
292 /// function's signature.
293 vm_context: Option<Reg>,
294 }
295
296 /// Notion of "relocation distance". This gives an estimate of how far away a symbol will be from a
297 /// reference.
298 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
299 pub enum RelocDistance {
300 /// Target of relocation is "nearby". The threshold for this is fuzzy but should be interpreted
301 /// as approximately "within the compiled output of one module"; e.g., within AArch64's +/-
302 /// 128MB offset. If unsure, use `Far` instead.
303 Near,
304 /// Target of relocation could be anywhere in the address space.
305 Far,
306 }
307
alloc_vregs<I: VCodeInst>( ty: Type, next_vreg: &mut u32, vcode: &mut VCodeBuilder<I>, ) -> CodegenResult<ValueRegs<Reg>>308 fn alloc_vregs<I: VCodeInst>(
309 ty: Type,
310 next_vreg: &mut u32,
311 vcode: &mut VCodeBuilder<I>,
312 ) -> CodegenResult<ValueRegs<Reg>> {
313 let v = *next_vreg;
314 let (regclasses, tys) = I::rc_for_type(ty)?;
315 *next_vreg += regclasses.len() as u32;
316 let regs = match regclasses {
317 &[rc0] => ValueRegs::one(Reg::new_virtual(rc0, v)),
318 &[rc0, rc1] => ValueRegs::two(Reg::new_virtual(rc0, v), Reg::new_virtual(rc1, v + 1)),
319 #[cfg(feature = "arm32")]
320 &[rc0, rc1, rc2, rc3] => ValueRegs::four(
321 Reg::new_virtual(rc0, v),
322 Reg::new_virtual(rc1, v + 1),
323 Reg::new_virtual(rc2, v + 2),
324 Reg::new_virtual(rc3, v + 3),
325 ),
326 _ => panic!("Value must reside in 1, 2 or 4 registers"),
327 };
328 for (®_ty, ®) in tys.iter().zip(regs.regs().iter()) {
329 vcode.set_vreg_type(reg.to_virtual_reg(), reg_ty);
330 }
331 Ok(regs)
332 }
333
334 enum GenerateReturn {
335 Yes,
336 No,
337 }
338
339 impl<'func, I: VCodeInst> Lower<'func, I> {
340 /// Prepare a new lowering context for the given IR function.
new( f: &'func Function, abi: Box<dyn ABICallee<I = I>>, emit_info: I::Info, block_order: BlockLoweringOrder, ) -> CodegenResult<Lower<'func, I>>341 pub fn new(
342 f: &'func Function,
343 abi: Box<dyn ABICallee<I = I>>,
344 emit_info: I::Info,
345 block_order: BlockLoweringOrder,
346 ) -> CodegenResult<Lower<'func, I>> {
347 let constants = VCodeConstants::with_capacity(f.dfg.constants.len());
348 let mut vcode = VCodeBuilder::new(abi, emit_info, block_order, constants);
349
350 let mut next_vreg: u32 = 0;
351
352 let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid());
353
354 // Assign a vreg to each block param and each inst result.
355 for bb in f.layout.blocks() {
356 for ¶m in f.dfg.block_params(bb) {
357 let ty = f.dfg.value_type(param);
358 if value_regs[param].is_invalid() {
359 let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
360 value_regs[param] = regs;
361 debug!("bb {} param {}: regs {:?}", bb, param, regs);
362 }
363 }
364 for inst in f.layout.block_insts(bb) {
365 for &result in f.dfg.inst_results(inst) {
366 let ty = f.dfg.value_type(result);
367 if value_regs[result].is_invalid() {
368 let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
369 value_regs[result] = regs;
370 debug!(
371 "bb {} inst {} ({:?}): result regs {:?}",
372 bb, inst, f.dfg[inst], regs,
373 );
374 }
375 }
376 }
377 }
378
379 let vm_context = vcode
380 .abi()
381 .signature()
382 .special_param_index(ArgumentPurpose::VMContext)
383 .map(|vm_context_index| {
384 let entry_block = f.layout.entry_block().unwrap();
385 let param = f.dfg.block_params(entry_block)[vm_context_index];
386 value_regs[param].only_reg().unwrap()
387 });
388
389 // Assign vreg(s) to each return value.
390 let mut retval_regs = vec![];
391 for ret in &vcode.abi().signature().returns.clone() {
392 let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?;
393 retval_regs.push(regs);
394 debug!("retval gets regs {:?}", regs);
395 }
396
397 // Compute instruction colors, find constant instructions, and find instructions with
398 // side-effects, in one combined pass.
399 let mut cur_color = 0;
400 let mut block_end_colors = SecondaryMap::with_default(InstColor::new(0));
401 let mut side_effect_inst_entry_colors = FxHashMap::default();
402 let mut inst_constants = FxHashMap::default();
403 let mut value_uses = SecondaryMap::with_default(0);
404 for bb in f.layout.blocks() {
405 cur_color += 1;
406 for inst in f.layout.block_insts(bb) {
407 let side_effect = has_lowering_side_effect(f, inst);
408
409 debug!("bb {} inst {} has color {}", bb, inst, cur_color);
410 if side_effect {
411 side_effect_inst_entry_colors.insert(inst, InstColor::new(cur_color));
412 debug!(" -> side-effecting; incrementing color for next inst");
413 cur_color += 1;
414 }
415
416 // Determine if this is a constant; if so, add to the table.
417 if let Some(c) = is_constant_64bit(f, inst) {
418 debug!(" -> constant: {}", c);
419 inst_constants.insert(inst, c);
420 }
421
422 // Count uses of all arguments.
423 for arg in f.dfg.inst_args(inst) {
424 let arg = f.dfg.resolve_aliases(*arg);
425 value_uses[arg] += 1;
426 }
427 }
428
429 block_end_colors[bb] = InstColor::new(cur_color);
430 }
431
432 Ok(Lower {
433 f,
434 vcode,
435 value_regs,
436 retval_regs,
437 block_end_colors,
438 side_effect_inst_entry_colors,
439 inst_constants,
440 next_vreg,
441 value_uses,
442 value_lowered_uses: SecondaryMap::default(),
443 inst_sunk: FxHashSet::default(),
444 cur_scan_entry_color: None,
445 cur_inst: None,
446 block_insts: vec![],
447 block_ranges: vec![],
448 bb_insts: vec![],
449 ir_insts: vec![],
450 pinned_reg: None,
451 vm_context,
452 })
453 }
454
gen_arg_setup(&mut self)455 fn gen_arg_setup(&mut self) {
456 if let Some(entry_bb) = self.f.layout.entry_block() {
457 debug!(
458 "gen_arg_setup: entry BB {} args are:\n{:?}",
459 entry_bb,
460 self.f.dfg.block_params(entry_bb)
461 );
462 for (i, param) in self.f.dfg.block_params(entry_bb).iter().enumerate() {
463 if !self.vcode.abi().arg_is_needed_in_body(i) {
464 continue;
465 }
466 let regs = writable_value_regs(self.value_regs[*param]);
467 for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() {
468 self.emit(insn);
469 }
470 if self.abi().signature().params[i].purpose == ArgumentPurpose::StructReturn {
471 assert!(regs.len() == 1);
472 let ty = self.abi().signature().params[i].value_type;
473 // The ABI implementation must have ensured that a StructReturn
474 // arg is present in the return values.
475 let struct_ret_idx = self
476 .abi()
477 .signature()
478 .returns
479 .iter()
480 .position(|ret| ret.purpose == ArgumentPurpose::StructReturn)
481 .expect("StructReturn return value not present!");
482 self.emit(I::gen_move(
483 Writable::from_reg(self.retval_regs[struct_ret_idx].regs()[0]),
484 regs.regs()[0].to_reg(),
485 ty,
486 ));
487 }
488 }
489 if let Some(insn) = self.vcode.abi().gen_retval_area_setup() {
490 self.emit(insn);
491 }
492 }
493 }
494
gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn)495 fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
496 // Hack: to keep `vmctx` alive, if it exists, we emit a value label here
497 // for it if debug info is requested. This ensures that it exists either
498 // in a register or spillslot throughout the entire function body, and
499 // allows for a better debugging experience.
500 if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) {
501 self.emit_value_label_marks_for_value(vmctx_val);
502 }
503
504 let retval_regs = self.retval_regs.clone();
505 for (i, regs) in retval_regs.into_iter().enumerate() {
506 let regs = writable_value_regs(regs);
507 for insn in self
508 .vcode
509 .abi()
510 .gen_copy_regs_to_retval(i, regs)
511 .into_iter()
512 {
513 self.emit(insn);
514 }
515 }
516 let inst = match gen_ret_inst {
517 GenerateReturn::Yes => self.vcode.abi().gen_ret(),
518 GenerateReturn::No => self.vcode.abi().gen_epilogue_placeholder(),
519 };
520 self.emit(inst);
521 }
522
lower_edge(&mut self, pred: Block, inst: Inst, succ: Block) -> CodegenResult<()>523 fn lower_edge(&mut self, pred: Block, inst: Inst, succ: Block) -> CodegenResult<()> {
524 debug!("lower_edge: pred {} succ {}", pred, succ);
525
526 let num_args = self.f.dfg.block_params(succ).len();
527 debug_assert!(num_args == self.f.dfg.inst_variable_args(inst).len());
528
529 // Most blocks have no params, so skip all the hoop-jumping below and make an early exit.
530 if num_args == 0 {
531 return Ok(());
532 }
533
534 self.cur_inst = Some(inst);
535
536 // Make up two vectors of info:
537 //
538 // * one for dsts which are to be assigned constants. We'll deal with those second, so
539 // as to minimise live ranges.
540 //
541 // * one for dsts whose sources are non-constants.
542
543 let mut const_bundles: SmallVec<[_; 16]> = SmallVec::new();
544 let mut var_bundles: SmallVec<[_; 16]> = SmallVec::new();
545
546 let mut i = 0;
547 for (dst_val, src_val) in self
548 .f
549 .dfg
550 .block_params(succ)
551 .iter()
552 .zip(self.f.dfg.inst_variable_args(inst).iter())
553 {
554 let src_val = self.f.dfg.resolve_aliases(*src_val);
555 let ty = self.f.dfg.value_type(src_val);
556
557 debug_assert!(ty == self.f.dfg.value_type(*dst_val));
558 let dst_regs = self.value_regs[*dst_val];
559
560 let input = self.get_value_as_source_or_const(src_val);
561 debug!("jump arg {} is {}", i, src_val);
562 i += 1;
563
564 if let Some(c) = input.constant {
565 debug!(" -> constant {}", c);
566 const_bundles.push((ty, writable_value_regs(dst_regs), c));
567 } else {
568 let src_regs = self.put_value_in_regs(src_val);
569 debug!(" -> reg {:?}", src_regs);
570 // Skip self-assignments. Not only are they pointless, they falsely trigger the
571 // overlap-check below and hence can cause a lot of unnecessary copying through
572 // temporaries.
573 if dst_regs != src_regs {
574 var_bundles.push((ty, writable_value_regs(dst_regs), src_regs));
575 }
576 }
577 }
578
579 // Deal first with the moves whose sources are variables.
580
581 // FIXME: use regalloc.rs' SparseSetU here. This would avoid all heap allocation
582 // for cases of up to circa 16 args. Currently not possible because regalloc.rs
583 // does not export it.
584 let mut src_reg_set = FxHashSet::<Reg>::default();
585 for (_, _, src_regs) in &var_bundles {
586 for ® in src_regs.regs() {
587 src_reg_set.insert(reg);
588 }
589 }
590 let mut overlaps = false;
591 'outer: for (_, dst_regs, _) in &var_bundles {
592 for ® in dst_regs.regs() {
593 if src_reg_set.contains(®.to_reg()) {
594 overlaps = true;
595 break 'outer;
596 }
597 }
598 }
599
600 // If, as is mostly the case, the source and destination register sets are non
601 // overlapping, then we can copy directly, so as to save the register allocator work.
602 if !overlaps {
603 for (ty, dst_regs, src_regs) in &var_bundles {
604 let (_, reg_tys) = I::rc_for_type(*ty)?;
605 for ((dst, src), reg_ty) in dst_regs
606 .regs()
607 .iter()
608 .zip(src_regs.regs().iter())
609 .zip(reg_tys.iter())
610 {
611 self.emit(I::gen_move(*dst, *src, *reg_ty));
612 }
613 }
614 } else {
615 // There's some overlap, so play safe and copy via temps.
616 let mut tmp_regs = SmallVec::<[ValueRegs<Writable<Reg>>; 16]>::new();
617 for (ty, _, _) in &var_bundles {
618 tmp_regs.push(self.alloc_tmp(*ty));
619 }
620 for ((ty, _, src_reg), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
621 let (_, reg_tys) = I::rc_for_type(*ty)?;
622 for ((tmp, src), reg_ty) in tmp_reg
623 .regs()
624 .iter()
625 .zip(src_reg.regs().iter())
626 .zip(reg_tys.iter())
627 {
628 self.emit(I::gen_move(*tmp, *src, *reg_ty));
629 }
630 }
631 for ((ty, dst_reg, _), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
632 let (_, reg_tys) = I::rc_for_type(*ty)?;
633 for ((dst, tmp), reg_ty) in dst_reg
634 .regs()
635 .iter()
636 .zip(tmp_reg.regs().iter())
637 .zip(reg_tys.iter())
638 {
639 self.emit(I::gen_move(*dst, tmp.to_reg(), *reg_ty));
640 }
641 }
642 }
643
644 // Now, finally, deal with the moves whose sources are constants.
645 for (ty, dst_reg, const_val) in &const_bundles {
646 for inst in I::gen_constant(*dst_reg, *const_val as u128, *ty, |ty| {
647 self.alloc_tmp(ty).only_reg().unwrap()
648 })
649 .into_iter()
650 {
651 self.emit(inst);
652 }
653 }
654
655 Ok(())
656 }
657
658 /// Has this instruction been sunk to a use-site (i.e., away from its
659 /// original location)?
is_inst_sunk(&self, inst: Inst) -> bool660 fn is_inst_sunk(&self, inst: Inst) -> bool {
661 self.inst_sunk.contains(&inst)
662 }
663
664 // Is any result of this instruction needed?
is_any_inst_result_needed(&self, inst: Inst) -> bool665 fn is_any_inst_result_needed(&self, inst: Inst) -> bool {
666 self.f
667 .dfg
668 .inst_results(inst)
669 .iter()
670 .any(|&result| self.value_lowered_uses[result] > 0)
671 }
672
lower_clif_block<B: LowerBackend<MInst = I>>( &mut self, backend: &B, block: Block, ) -> CodegenResult<()>673 fn lower_clif_block<B: LowerBackend<MInst = I>>(
674 &mut self,
675 backend: &B,
676 block: Block,
677 ) -> CodegenResult<()> {
678 self.cur_scan_entry_color = Some(self.block_end_colors[block]);
679 // Lowering loop:
680 // - For each non-branch instruction, in reverse order:
681 // - If side-effecting (load, store, branch/call/return, possible trap), or if
682 // used outside of this block, or if demanded by another inst, then lower.
683 //
684 // That's it! Lowering of side-effecting ops will force all *needed*
685 // (live) non-side-effecting ops to be lowered at the right places, via
686 // the `use_input_reg()` callback on the `LowerCtx` (that's us). That's
687 // because `use_input_reg()` sets the eager/demand bit for any insts
688 // whose result registers are used.
689 //
690 // We build up the BB in reverse instruction order in `bb_insts`.
691 // Because the machine backend calls `ctx.emit()` in forward order, we
692 // collect per-IR-inst lowered instructions in `ir_insts`, then reverse
693 // these and append to `bb_insts` as we go backward through the block.
694 // `bb_insts` are then reversed again and appended to the VCode at the
695 // end of the BB (in the toplevel driver `lower()`).
696 for inst in self.f.layout.block_insts(block).rev() {
697 let data = &self.f.dfg[inst];
698 let has_side_effect = has_lowering_side_effect(self.f, inst);
699 // If inst has been sunk to another location, skip it.
700 if self.is_inst_sunk(inst) {
701 continue;
702 }
703 // Are any outputs used at least once?
704 let value_needed = self.is_any_inst_result_needed(inst);
705 debug!(
706 "lower_clif_block: block {} inst {} ({:?}) is_branch {} side_effect {} value_needed {}",
707 block,
708 inst,
709 data,
710 data.opcode().is_branch(),
711 has_side_effect,
712 value_needed,
713 );
714
715 // Update scan state to color prior to this inst (as we are scanning
716 // backward).
717 self.cur_inst = Some(inst);
718 if has_side_effect {
719 let entry_color = *self
720 .side_effect_inst_entry_colors
721 .get(&inst)
722 .expect("every side-effecting inst should have a color-map entry");
723 self.cur_scan_entry_color = Some(entry_color);
724 }
725
726 // Skip lowering branches; these are handled separately
727 // (see `lower_clif_branches()` below).
728 if self.f.dfg[inst].opcode().is_branch() {
729 continue;
730 }
731
732 // Normal instruction: codegen if the instruction is side-effecting
733 // or any of its outputs its used.
734 if has_side_effect || value_needed {
735 debug!("lowering: inst {}: {:?}", inst, self.f.dfg[inst]);
736 backend.lower(self, inst)?;
737 // Emit value-label markers if needed, to later recover debug
738 // mappings.
739 self.emit_value_label_markers_for_inst(inst);
740 }
741 if data.opcode().is_return() {
742 // Return: handle specially, using ABI-appropriate sequence.
743 let gen_ret = if data.opcode() == Opcode::Return {
744 GenerateReturn::Yes
745 } else {
746 debug_assert!(data.opcode() == Opcode::FallthroughReturn);
747 GenerateReturn::No
748 };
749 self.gen_retval_setup(gen_ret);
750 }
751
752 let loc = self.srcloc(inst);
753 self.finish_ir_inst(loc);
754 }
755 self.cur_scan_entry_color = None;
756 Ok(())
757 }
758
get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]>759 fn get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]> {
760 if let Some(ref values_labels) = self.f.dfg.values_labels {
761 debug!(
762 "get_value_labels: val {} -> {} -> {:?}",
763 val,
764 self.f.dfg.resolve_aliases(val),
765 values_labels.get(&self.f.dfg.resolve_aliases(val))
766 );
767 let val = self.f.dfg.resolve_aliases(val);
768 match values_labels.get(&val) {
769 Some(&ValueLabelAssignments::Starts(ref list)) => Some(&list[..]),
770 Some(&ValueLabelAssignments::Alias { value, .. }) if depth < 10 => {
771 self.get_value_labels(value, depth + 1)
772 }
773 _ => None,
774 }
775 } else {
776 None
777 }
778 }
779
emit_value_label_marks_for_value(&mut self, val: Value)780 fn emit_value_label_marks_for_value(&mut self, val: Value) {
781 let mut markers: SmallVec<[I; 4]> = smallvec![];
782 let regs = self.value_regs[val];
783 if regs.len() > 1 {
784 return;
785 }
786 let reg = regs.only_reg().unwrap();
787
788 if let Some(label_starts) = self.get_value_labels(val, 0) {
789 let labels = label_starts
790 .iter()
791 .map(|&ValueLabelStart { label, .. }| label)
792 .collect::<FxHashSet<_>>();
793 for label in labels {
794 debug!(
795 "value labeling: defines val {:?} -> reg {:?} -> label {:?}",
796 val, reg, label,
797 );
798 markers.push(I::gen_value_label_marker(label, reg));
799 }
800 }
801 for marker in markers {
802 self.emit(marker);
803 }
804 }
805
emit_value_label_markers_for_inst(&mut self, inst: Inst)806 fn emit_value_label_markers_for_inst(&mut self, inst: Inst) {
807 if self.f.dfg.values_labels.is_none() {
808 return;
809 }
810
811 debug!(
812 "value labeling: srcloc {}: inst {}",
813 self.srcloc(inst),
814 inst
815 );
816 for &val in self.f.dfg.inst_results(inst) {
817 self.emit_value_label_marks_for_value(val);
818 }
819 }
820
emit_value_label_markers_for_block_args(&mut self, block: Block)821 fn emit_value_label_markers_for_block_args(&mut self, block: Block) {
822 if self.f.dfg.values_labels.is_none() {
823 return;
824 }
825
826 debug!("value labeling: block {}", block);
827 for &arg in self.f.dfg.block_params(block) {
828 self.emit_value_label_marks_for_value(arg);
829 }
830 self.finish_ir_inst(SourceLoc::default());
831 }
832
finish_ir_inst(&mut self, loc: SourceLoc)833 fn finish_ir_inst(&mut self, loc: SourceLoc) {
834 // `bb_insts` is kept in reverse order, so emit the instructions in
835 // reverse order.
836 for mut tuple in self.ir_insts.drain(..).rev() {
837 tuple.loc = loc;
838 self.bb_insts.push(tuple);
839 }
840 }
841
finish_bb(&mut self)842 fn finish_bb(&mut self) {
843 let start = self.block_insts.len();
844 for tuple in self.bb_insts.drain(..).rev() {
845 self.block_insts.push(tuple);
846 }
847 let end = self.block_insts.len();
848 self.block_ranges.push((start, end));
849 }
850
copy_bbs_to_vcode(&mut self)851 fn copy_bbs_to_vcode(&mut self) {
852 for &(start, end) in self.block_ranges.iter().rev() {
853 for &InstTuple {
854 loc,
855 is_safepoint,
856 ref inst,
857 } in &self.block_insts[start..end]
858 {
859 self.vcode.set_srcloc(loc);
860 self.vcode.push(inst.clone(), is_safepoint);
861 }
862 self.vcode.end_bb();
863 }
864 }
865
lower_clif_branches<B: LowerBackend<MInst = I>>( &mut self, backend: &B, block: Block, branches: &SmallVec<[Inst; 2]>, targets: &SmallVec<[MachLabel; 2]>, ) -> CodegenResult<()>866 fn lower_clif_branches<B: LowerBackend<MInst = I>>(
867 &mut self,
868 backend: &B,
869 block: Block,
870 branches: &SmallVec<[Inst; 2]>,
871 targets: &SmallVec<[MachLabel; 2]>,
872 ) -> CodegenResult<()> {
873 debug!(
874 "lower_clif_branches: block {} branches {:?} targets {:?}",
875 block, branches, targets,
876 );
877 // When considering code-motion opportunities, consider the current
878 // program point to be the first branch.
879 self.cur_inst = Some(branches[0]);
880 backend.lower_branch_group(self, branches, targets)?;
881 let loc = self.srcloc(branches[0]);
882 self.finish_ir_inst(loc);
883 Ok(())
884 }
885
collect_branches_and_targets( &self, bindex: BlockIndex, _bb: Block, branches: &mut SmallVec<[Inst; 2]>, targets: &mut SmallVec<[MachLabel; 2]>, )886 fn collect_branches_and_targets(
887 &self,
888 bindex: BlockIndex,
889 _bb: Block,
890 branches: &mut SmallVec<[Inst; 2]>,
891 targets: &mut SmallVec<[MachLabel; 2]>,
892 ) {
893 branches.clear();
894 targets.clear();
895 let mut last_inst = None;
896 for &(inst, succ) in self.vcode.block_order().succ_indices(bindex) {
897 // Avoid duplicates: this ensures a br_table is only inserted once.
898 if last_inst != Some(inst) {
899 branches.push(inst);
900 } else {
901 debug_assert!(self.f.dfg[inst].opcode() == Opcode::BrTable);
902 debug_assert!(branches.len() == 1);
903 }
904 last_inst = Some(inst);
905 targets.push(MachLabel::from_block(succ));
906 }
907 }
908
909 /// Lower the function.
lower<B: LowerBackend<MInst = I>>( mut self, backend: &B, ) -> CodegenResult<(VCode<I>, StackmapRequestInfo)>910 pub fn lower<B: LowerBackend<MInst = I>>(
911 mut self,
912 backend: &B,
913 ) -> CodegenResult<(VCode<I>, StackmapRequestInfo)> {
914 debug!("about to lower function: {:?}", self.f);
915
916 // Initialize the ABI object, giving it a temp if requested.
917 let maybe_tmp = if let Some(temp_ty) = self.vcode.abi().temp_needed() {
918 Some(self.alloc_tmp(temp_ty).only_reg().unwrap())
919 } else {
920 None
921 };
922 self.vcode.abi().init(maybe_tmp);
923
924 // Get the pinned reg here (we only parameterize this function on `B`,
925 // not the whole `Lower` impl).
926 self.pinned_reg = backend.maybe_pinned_reg();
927
928 self.vcode.set_entry(0);
929
930 // Reused vectors for branch lowering.
931 let mut branches: SmallVec<[Inst; 2]> = SmallVec::new();
932 let mut targets: SmallVec<[MachLabel; 2]> = SmallVec::new();
933
934 // get a copy of the lowered order; we hold this separately because we
935 // need a mut ref to the vcode to mutate it below.
936 let lowered_order: SmallVec<[LoweredBlock; 64]> = self
937 .vcode
938 .block_order()
939 .lowered_order()
940 .iter()
941 .cloned()
942 .collect();
943
944 // Main lowering loop over lowered blocks.
945 for (bindex, lb) in lowered_order.iter().enumerate().rev() {
946 let bindex = bindex as BlockIndex;
947
948 // Lower the block body in reverse order (see comment in
949 // `lower_clif_block()` for rationale).
950
951 // End branches.
952 if let Some(bb) = lb.orig_block() {
953 self.collect_branches_and_targets(bindex, bb, &mut branches, &mut targets);
954 if branches.len() > 0 {
955 self.lower_clif_branches(backend, bb, &branches, &targets)?;
956 self.finish_ir_inst(self.srcloc(branches[0]));
957 }
958 } else {
959 // If no orig block, this must be a pure edge block; get the successor and
960 // emit a jump.
961 let (_, succ) = self.vcode.block_order().succ_indices(bindex)[0];
962 self.emit(I::gen_jump(MachLabel::from_block(succ)));
963 self.finish_ir_inst(SourceLoc::default());
964 }
965
966 // Out-edge phi moves.
967 if let Some((pred, inst, succ)) = lb.out_edge() {
968 self.lower_edge(pred, inst, succ)?;
969 self.finish_ir_inst(SourceLoc::default());
970 }
971 // Original block body.
972 if let Some(bb) = lb.orig_block() {
973 self.lower_clif_block(backend, bb)?;
974 self.emit_value_label_markers_for_block_args(bb);
975 }
976 // In-edge phi moves.
977 if let Some((pred, inst, succ)) = lb.in_edge() {
978 self.lower_edge(pred, inst, succ)?;
979 self.finish_ir_inst(SourceLoc::default());
980 }
981
982 if bindex == 0 {
983 // Set up the function with arg vreg inits.
984 self.gen_arg_setup();
985 self.finish_ir_inst(SourceLoc::default());
986 }
987
988 self.finish_bb();
989 }
990
991 self.copy_bbs_to_vcode();
992
993 // Now that we've emitted all instructions into the VCodeBuilder, let's build the VCode.
994 let (vcode, stack_map_info) = self.vcode.build();
995 debug!("built vcode: {:?}", vcode);
996
997 Ok((vcode, stack_map_info))
998 }
999
put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg>1000 fn put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg> {
1001 debug!("put_value_in_reg: val {}", val);
1002 let mut regs = self.value_regs[val];
1003 debug!(" -> regs {:?}", regs);
1004 assert!(regs.is_valid());
1005
1006 self.value_lowered_uses[val] += 1;
1007
1008 // Pinned-reg hack: if backend specifies a fixed pinned register, use it
1009 // directly when we encounter a GetPinnedReg op, rather than lowering
1010 // the actual op, and do not return the source inst to the caller; the
1011 // value comes "out of the ether" and we will not force generation of
1012 // the superfluous move.
1013 if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) {
1014 if self.f.dfg[i].opcode() == Opcode::GetPinnedReg {
1015 if let Some(pr) = self.pinned_reg {
1016 regs = ValueRegs::one(pr);
1017 }
1018 }
1019 }
1020
1021 regs
1022 }
1023
1024 /// Get the actual inputs for a value. This is the implementation for
1025 /// `get_input()` but starting from the SSA value, which is not exposed to
1026 /// the backend.
get_value_as_source_or_const(&self, val: Value) -> NonRegInput1027 fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput {
1028 debug!(
1029 "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}",
1030 val, self.cur_inst, self.cur_scan_entry_color,
1031 );
1032 let inst = match self.f.dfg.value_def(val) {
1033 // OK to merge source instruction if (i) we have a source
1034 // instruction, and:
1035 // - It has no side-effects, OR
1036 // - It has a side-effect, has one output value, that one output has
1037 // only one use (this one), and the instruction's color is *one less
1038 // than* the current scan color.
1039 //
1040 // This latter set of conditions is testing whether a
1041 // side-effecting instruction can sink to the current scan
1042 // location; this is possible if the in-color of this inst is
1043 // equal to the out-color of the producing inst, so no other
1044 // side-effecting ops occur between them (which will only be true
1045 // if they are in the same BB, because color increments at each BB
1046 // start).
1047 //
1048 // If it is actually sunk, then in `merge_inst()`, we update the
1049 // scan color so that as we scan over the range past which the
1050 // instruction was sunk, we allow other instructions (that came
1051 // prior to the sunk instruction) to sink.
1052 ValueDef::Result(src_inst, result_idx) => {
1053 let src_side_effect = has_lowering_side_effect(self.f, src_inst);
1054 debug!(" -> src inst {}", src_inst);
1055 debug!(" -> has lowering side effect: {}", src_side_effect);
1056 if !src_side_effect {
1057 // Pure instruction: always possible to sink.
1058 Some((src_inst, result_idx))
1059 } else {
1060 // Side-effect: test whether this is the only use of the
1061 // only result of the instruction, and whether colors allow
1062 // the code-motion.
1063 if self.cur_scan_entry_color.is_some()
1064 && self.value_uses[val] == 1
1065 && self.value_lowered_uses[val] == 0
1066 && self.num_outputs(src_inst) == 1
1067 && self
1068 .side_effect_inst_entry_colors
1069 .get(&src_inst)
1070 .unwrap()
1071 .get()
1072 + 1
1073 == self.cur_scan_entry_color.unwrap().get()
1074 {
1075 Some((src_inst, 0))
1076 } else {
1077 None
1078 }
1079 }
1080 }
1081 _ => None,
1082 };
1083 let constant = inst.and_then(|(inst, _)| self.get_constant(inst));
1084
1085 NonRegInput { inst, constant }
1086 }
1087 }
1088
1089 impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
1090 type I = I;
1091
abi(&mut self) -> &mut dyn ABICallee<I = I>1092 fn abi(&mut self) -> &mut dyn ABICallee<I = I> {
1093 self.vcode.abi()
1094 }
1095
retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>1096 fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>> {
1097 writable_value_regs(self.retval_regs[idx])
1098 }
1099
get_vm_context(&self) -> Option<Reg>1100 fn get_vm_context(&self) -> Option<Reg> {
1101 self.vm_context
1102 }
1103
data(&self, ir_inst: Inst) -> &InstructionData1104 fn data(&self, ir_inst: Inst) -> &InstructionData {
1105 &self.f.dfg[ir_inst]
1106 }
1107
ty(&self, ir_inst: Inst) -> Type1108 fn ty(&self, ir_inst: Inst) -> Type {
1109 self.f.dfg.ctrl_typevar(ir_inst)
1110 }
1111
call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>1112 fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> {
1113 match &self.f.dfg[ir_inst] {
1114 &InstructionData::Call { func_ref, .. }
1115 | &InstructionData::FuncAddr { func_ref, .. } => {
1116 let funcdata = &self.f.dfg.ext_funcs[func_ref];
1117 let dist = funcdata.reloc_distance();
1118 Some((&funcdata.name, dist))
1119 }
1120 _ => None,
1121 }
1122 }
1123
call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>1124 fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> {
1125 match &self.f.dfg[ir_inst] {
1126 &InstructionData::Call { func_ref, .. } => {
1127 let funcdata = &self.f.dfg.ext_funcs[func_ref];
1128 Some(&self.f.dfg.signatures[funcdata.signature])
1129 }
1130 &InstructionData::CallIndirect { sig_ref, .. } => Some(&self.f.dfg.signatures[sig_ref]),
1131 _ => None,
1132 }
1133 }
1134
symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>1135 fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)> {
1136 match &self.f.dfg[ir_inst] {
1137 &InstructionData::UnaryGlobalValue { global_value, .. } => {
1138 let gvdata = &self.f.global_values[global_value];
1139 match gvdata {
1140 &GlobalValueData::Symbol {
1141 ref name,
1142 ref offset,
1143 ..
1144 } => {
1145 let offset = offset.bits();
1146 let dist = gvdata.maybe_reloc_distance().unwrap();
1147 Some((name, dist, offset))
1148 }
1149 _ => None,
1150 }
1151 }
1152 _ => None,
1153 }
1154 }
1155
memflags(&self, ir_inst: Inst) -> Option<MemFlags>1156 fn memflags(&self, ir_inst: Inst) -> Option<MemFlags> {
1157 match &self.f.dfg[ir_inst] {
1158 &InstructionData::AtomicCas { flags, .. } => Some(flags),
1159 &InstructionData::AtomicRmw { flags, .. } => Some(flags),
1160 &InstructionData::Load { flags, .. }
1161 | &InstructionData::LoadComplex { flags, .. }
1162 | &InstructionData::LoadNoOffset { flags, .. }
1163 | &InstructionData::Store { flags, .. }
1164 | &InstructionData::StoreComplex { flags, .. } => Some(flags),
1165 &InstructionData::StoreNoOffset { flags, .. } => Some(flags),
1166 _ => None,
1167 }
1168 }
1169
srcloc(&self, ir_inst: Inst) -> SourceLoc1170 fn srcloc(&self, ir_inst: Inst) -> SourceLoc {
1171 self.f.srclocs[ir_inst]
1172 }
1173
num_inputs(&self, ir_inst: Inst) -> usize1174 fn num_inputs(&self, ir_inst: Inst) -> usize {
1175 self.f.dfg.inst_args(ir_inst).len()
1176 }
1177
num_outputs(&self, ir_inst: Inst) -> usize1178 fn num_outputs(&self, ir_inst: Inst) -> usize {
1179 self.f.dfg.inst_results(ir_inst).len()
1180 }
1181
input_ty(&self, ir_inst: Inst, idx: usize) -> Type1182 fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type {
1183 let val = self.f.dfg.inst_args(ir_inst)[idx];
1184 let val = self.f.dfg.resolve_aliases(val);
1185 self.f.dfg.value_type(val)
1186 }
1187
output_ty(&self, ir_inst: Inst, idx: usize) -> Type1188 fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type {
1189 self.f.dfg.value_type(self.f.dfg.inst_results(ir_inst)[idx])
1190 }
1191
get_constant(&self, ir_inst: Inst) -> Option<u64>1192 fn get_constant(&self, ir_inst: Inst) -> Option<u64> {
1193 self.inst_constants.get(&ir_inst).cloned()
1194 }
1195
get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput1196 fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput {
1197 let val = self.f.dfg.inst_args(ir_inst)[idx];
1198 let val = self.f.dfg.resolve_aliases(val);
1199 self.get_value_as_source_or_const(val)
1200 }
1201
put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>1202 fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg> {
1203 let val = self.f.dfg.inst_args(ir_inst)[idx];
1204 let val = self.f.dfg.resolve_aliases(val);
1205 self.put_value_in_regs(val)
1206 }
1207
get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>1208 fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>> {
1209 let val = self.f.dfg.inst_results(ir_inst)[idx];
1210 writable_value_regs(self.value_regs[val])
1211 }
1212
alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>1213 fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>> {
1214 writable_value_regs(alloc_vregs(ty, &mut self.next_vreg, &mut self.vcode).unwrap())
1215 }
1216
emit(&mut self, mach_inst: I)1217 fn emit(&mut self, mach_inst: I) {
1218 self.ir_insts.push(InstTuple {
1219 loc: SourceLoc::default(),
1220 is_safepoint: false,
1221 inst: mach_inst,
1222 });
1223 }
1224
emit_safepoint(&mut self, mach_inst: I)1225 fn emit_safepoint(&mut self, mach_inst: I) {
1226 self.ir_insts.push(InstTuple {
1227 loc: SourceLoc::default(),
1228 is_safepoint: true,
1229 inst: mach_inst,
1230 });
1231 }
1232
sink_inst(&mut self, ir_inst: Inst)1233 fn sink_inst(&mut self, ir_inst: Inst) {
1234 assert!(has_lowering_side_effect(self.f, ir_inst));
1235 assert!(self.cur_scan_entry_color.is_some());
1236
1237 let sunk_inst_entry_color = self
1238 .side_effect_inst_entry_colors
1239 .get(&ir_inst)
1240 .cloned()
1241 .unwrap();
1242 let sunk_inst_exit_color = InstColor::new(sunk_inst_entry_color.get() + 1);
1243 assert!(sunk_inst_exit_color == self.cur_scan_entry_color.unwrap());
1244 self.cur_scan_entry_color = Some(sunk_inst_entry_color);
1245 self.inst_sunk.insert(ir_inst);
1246 }
1247
get_constant_data(&self, constant_handle: Constant) -> &ConstantData1248 fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData {
1249 self.f.dfg.constants.get(constant_handle)
1250 }
1251
use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant1252 fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant {
1253 self.vcode.constants().insert(constant)
1254 }
1255
get_immediate(&self, ir_inst: Inst) -> Option<DataValue>1256 fn get_immediate(&self, ir_inst: Inst) -> Option<DataValue> {
1257 let inst_data = self.data(ir_inst);
1258 match inst_data {
1259 InstructionData::Shuffle { mask, .. } => {
1260 let buffer = self.f.dfg.immediates.get(mask.clone()).unwrap().as_slice();
1261 let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer"));
1262 Some(value)
1263 }
1264 InstructionData::UnaryConst {
1265 constant_handle, ..
1266 } => {
1267 let buffer = self.f.dfg.constants.get(constant_handle.clone()).as_slice();
1268 let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer"));
1269 Some(value)
1270 }
1271 _ => inst_data.imm_value(),
1272 }
1273 }
1274
ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg1275 fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg {
1276 if reg.is_virtual() {
1277 reg
1278 } else {
1279 let new_reg = self.alloc_tmp(ty).only_reg().unwrap();
1280 self.emit(I::gen_move(new_reg, reg, ty));
1281 new_reg.to_reg()
1282 }
1283 }
1284 }
1285
1286 /// Visit all successors of a block with a given visitor closure.
visit_block_succs<F: FnMut(Inst, Block)>(f: &Function, block: Block, mut visit: F)1287 pub(crate) fn visit_block_succs<F: FnMut(Inst, Block)>(f: &Function, block: Block, mut visit: F) {
1288 for inst in f.layout.block_likely_branches(block) {
1289 if f.dfg[inst].opcode().is_branch() {
1290 visit_branch_targets(f, block, inst, &mut visit);
1291 }
1292 }
1293 }
1294
visit_branch_targets<F: FnMut(Inst, Block)>( f: &Function, block: Block, inst: Inst, visit: &mut F, )1295 fn visit_branch_targets<F: FnMut(Inst, Block)>(
1296 f: &Function,
1297 block: Block,
1298 inst: Inst,
1299 visit: &mut F,
1300 ) {
1301 if f.dfg[inst].opcode() == Opcode::Fallthrough {
1302 visit(inst, f.layout.next_block(block).unwrap());
1303 } else {
1304 match f.dfg[inst].analyze_branch(&f.dfg.value_lists) {
1305 BranchInfo::NotABranch => {}
1306 BranchInfo::SingleDest(dest, _) => {
1307 visit(inst, dest);
1308 }
1309 BranchInfo::Table(table, maybe_dest) => {
1310 if let Some(dest) = maybe_dest {
1311 visit(inst, dest);
1312 }
1313 for &dest in f.jump_tables[table].as_slice() {
1314 visit(inst, dest);
1315 }
1316 }
1317 }
1318 }
1319 }
1320