1 //! This module implements lowering (instruction selection) from Cranelift IR
2 //! to machine instructions with virtual registers. This is *almost* the final
3 //! machine code, except for register allocation.
4
5 // TODO: separate the IR-query core of `LowerCtx` from the lowering logic built
6 // on top of it, e.g. the side-effect/coloring analysis and the scan support.
7
8 use crate::data_value::DataValue;
9 use crate::entity::SecondaryMap;
10 use crate::fx::{FxHashMap, FxHashSet};
11 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
12 use crate::ir::instructions::BranchInfo;
13 use crate::ir::{
14 ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
15 InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
16 ValueLabelAssignments, ValueLabelStart,
17 };
18 use crate::machinst::{
19 writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
20 VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs,
21 };
22 use crate::CodegenResult;
23 use alloc::boxed::Box;
24 use alloc::vec::Vec;
25 use core::convert::TryInto;
26 use regalloc::{Reg, StackmapRequestInfo, Writable};
27 use smallvec::{smallvec, SmallVec};
28 use std::fmt::Debug;
29
30 /// An "instruction color" partitions CLIF instructions by side-effecting ops.
31 /// All instructions with the same "color" are guaranteed not to be separated by
32 /// any side-effecting op (for this purpose, loads are also considered
33 /// side-effecting, to avoid subtle questions w.r.t. the memory model), and
34 /// furthermore, it is guaranteed that for any two instructions A and B such
35 /// that color(A) == color(B), either A dominates B and B postdominates A, or
36 /// vice-versa. (For now, in practice, only ops in the same basic block can ever
37 /// have the same color, trivially providing the second condition.) Intuitively,
38 /// this means that the ops of the same color must always execute "together", as
39 /// part of one atomic contiguous section of the dynamic execution trace, and
40 /// they can be freely permuted (modulo true dataflow dependencies) without
41 /// affecting program behavior.
42 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
43 struct InstColor(u32);
44 impl InstColor {
new(n: u32) -> InstColor45 fn new(n: u32) -> InstColor {
46 InstColor(n)
47 }
48
49 /// Get an arbitrary index representing this color. The index is unique
50 /// *within a single function compilation*, but indices may be reused across
51 /// functions.
get(self) -> u3252 pub fn get(self) -> u32 {
53 self.0
54 }
55 }
56
57 /// A context that machine-specific lowering code can use to emit lowered
58 /// instructions. This is the view of the machine-independent per-function
59 /// lowering context that is seen by the machine backend.
60 pub trait LowerCtx {
61 /// The instruction type for which this lowering framework is instantiated.
62 type I: VCodeInst;
63
64 // Function-level queries:
65
66 /// Get the `ABICallee`.
abi(&mut self) -> &mut dyn ABICallee<I = Self::I>67 fn abi(&mut self) -> &mut dyn ABICallee<I = Self::I>;
68 /// Get the (virtual) register that receives the return value. A return
69 /// instruction should lower into a sequence that fills this register. (Why
70 /// not allow the backend to specify its own result register for the return?
71 /// Because there may be multiple return points.)
retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>72 fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>;
73 /// Returns the vreg containing the VmContext parameter, if there's one.
get_vm_context(&self) -> Option<Reg>74 fn get_vm_context(&self) -> Option<Reg>;
75
76 // General instruction queries:
77
78 /// Get the instdata for a given IR instruction.
data(&self, ir_inst: Inst) -> &InstructionData79 fn data(&self, ir_inst: Inst) -> &InstructionData;
80 /// Get the controlling type for a polymorphic IR instruction.
ty(&self, ir_inst: Inst) -> Type81 fn ty(&self, ir_inst: Inst) -> Type;
82 /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple
83 /// providing this name and the "relocation distance", i.e., whether the backend
84 /// can assume the target will be "nearby" (within some small offset) or an
85 /// arbitrary address. (This comes from the `colocated` bit in the CLIF.)
call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>86 fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>;
87 /// Get the signature for a call or call-indirect instruction.
call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>88 fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>;
89 /// Get the symbol name, relocation distance estimate, and offset for a
90 /// symbol_value instruction.
symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>91 fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>;
92 /// Returns the memory flags of a given memory access.
memflags(&self, ir_inst: Inst) -> Option<MemFlags>93 fn memflags(&self, ir_inst: Inst) -> Option<MemFlags>;
94 /// Get the source location for a given instruction.
srcloc(&self, ir_inst: Inst) -> SourceLoc95 fn srcloc(&self, ir_inst: Inst) -> SourceLoc;
96
97 // Instruction input/output queries:
98
99 /// Get the number of inputs to the given IR instruction.
num_inputs(&self, ir_inst: Inst) -> usize100 fn num_inputs(&self, ir_inst: Inst) -> usize;
101 /// Get the number of outputs to the given IR instruction.
num_outputs(&self, ir_inst: Inst) -> usize102 fn num_outputs(&self, ir_inst: Inst) -> usize;
103 /// Get the type for an instruction's input.
input_ty(&self, ir_inst: Inst, idx: usize) -> Type104 fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type;
105 /// Get the type for an instruction's output.
output_ty(&self, ir_inst: Inst, idx: usize) -> Type106 fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type;
107 /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit
108 /// value, if possible.
get_constant(&self, ir_inst: Inst) -> Option<u64>109 fn get_constant(&self, ir_inst: Inst) -> Option<u64>;
110 /// Get the input as one of two options other than a direct register:
111 ///
112 /// - An instruction, given that it is effect-free or able to sink its
113 /// effect to the current instruction being lowered, and given it has only
114 /// one output, and if effect-ful, given that this is the only use;
115 /// - A constant, if the value is a constant.
116 ///
117 /// The instruction input may be available in either of these forms. It may
118 /// be available in neither form, if the conditions are not met; if so, use
119 /// `put_input_in_regs()` instead to get it in a register.
120 ///
121 /// If the backend merges the effect of a side-effecting instruction, it
122 /// must call `sink_inst()`. When this is called, it indicates that the
123 /// effect has been sunk to the current scan location. The sunk
124 /// instruction's result(s) must have *no* uses remaining, because it will
125 /// not be codegen'd (it has been integrated into the current instruction).
get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput126 fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput;
127 /// Put the `idx`th input into register(s) and return the assigned register.
put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>128 fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>;
129 /// Get the `idx`th output register(s) of the given IR instruction. When
130 /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that
131 /// the backend will write results to these output register(s). This
132 /// register will always be "fresh"; it is guaranteed not to overlap with
133 /// any of the inputs, and can be freely used as a scratch register within
134 /// the lowered instruction sequence, as long as its final value is the
135 /// result of the computation.
get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>136 fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>;
137
138 // Codegen primitives: allocate temps, emit instructions, set result registers,
139 // ask for an input to be gen'd into a register.
140
141 /// Get a new temp.
alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>142 fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>;
143 /// Emit a machine instruction.
emit(&mut self, mach_inst: Self::I)144 fn emit(&mut self, mach_inst: Self::I);
145 /// Emit a machine instruction that is a safepoint.
emit_safepoint(&mut self, mach_inst: Self::I)146 fn emit_safepoint(&mut self, mach_inst: Self::I);
147 /// Indicate that the side-effect of an instruction has been sunk to the
148 /// current scan location. This should only be done with the instruction's
149 /// original results are not used (i.e., `put_input_in_regs` is not invoked
150 /// for the input produced by the sunk instruction), otherwise the
151 /// side-effect will occur twice.
sink_inst(&mut self, ir_inst: Inst)152 fn sink_inst(&mut self, ir_inst: Inst);
153 /// Retrieve constant data given a handle.
get_constant_data(&self, constant_handle: Constant) -> &ConstantData154 fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData;
155 /// Indicate that a constant should be emitted.
use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant156 fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant;
157 /// Retrieve the value immediate from an instruction. This will perform necessary lookups on the
158 /// `DataFlowGraph` to retrieve even large immediates.
get_immediate(&self, ir_inst: Inst) -> Option<DataValue>159 fn get_immediate(&self, ir_inst: Inst) -> Option<DataValue>;
160 /// Cause the value in `reg` to be in a virtual reg, by copying it into a new virtual reg
161 /// if `reg` is a real reg. `ty` describes the type of the value in `reg`.
ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg162 fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg;
163 }
164
165 /// A representation of all of the ways in which a value is available, aside
166 /// from as a direct register.
167 ///
168 /// - An instruction, if it would be allowed to occur at the current location
169 /// instead (see [LowerCtx::get_input_as_source_or_const()] for more
170 /// details).
171 ///
172 /// - A constant, if the value is known to be a constant.
173 #[derive(Clone, Copy, Debug)]
174 pub struct NonRegInput {
175 /// An instruction produces this value (as the given output), and its
176 /// computation (and side-effect if applicable) could occur at the
177 /// current instruction's location instead.
178 ///
179 /// If this instruction's operation is merged into the current instruction,
180 /// the backend must call [LowerCtx::sink_inst()].
181 pub inst: Option<(Inst, usize)>,
182 /// The value is a known constant.
183 pub constant: Option<u64>,
184 }
185
186 /// A machine backend.
187 pub trait LowerBackend {
188 /// The machine instruction type.
189 type MInst: VCodeInst;
190
191 /// Lower a single instruction.
192 ///
193 /// For a branch, this function should not generate the actual branch
194 /// instruction. However, it must force any values it needs for the branch
195 /// edge (block-param actuals) into registers, because the actual branch
196 /// generation (`lower_branch_group()`) happens *after* any possible merged
197 /// out-edge.
lower<C: LowerCtx<I = Self::MInst>>(&self, ctx: &mut C, inst: Inst) -> CodegenResult<()>198 fn lower<C: LowerCtx<I = Self::MInst>>(&self, ctx: &mut C, inst: Inst) -> CodegenResult<()>;
199
200 /// Lower a block-terminating group of branches (which together can be seen
201 /// as one N-way branch), given a vcode MachLabel for each target.
lower_branch_group<C: LowerCtx<I = Self::MInst>>( &self, ctx: &mut C, insts: &[Inst], targets: &[MachLabel], ) -> CodegenResult<()>202 fn lower_branch_group<C: LowerCtx<I = Self::MInst>>(
203 &self,
204 ctx: &mut C,
205 insts: &[Inst],
206 targets: &[MachLabel],
207 ) -> CodegenResult<()>;
208
209 /// A bit of a hack: give a fixed register that always holds the result of a
210 /// `get_pinned_reg` instruction, if known. This allows elision of moves
211 /// into the associated vreg, instead using the real reg directly.
maybe_pinned_reg(&self) -> Option<Reg>212 fn maybe_pinned_reg(&self) -> Option<Reg> {
213 None
214 }
215 }
216
217 /// A pending instruction to insert and auxiliary information about it: its source location and
218 /// whether it is a safepoint.
219 struct InstTuple<I: VCodeInst> {
220 loc: SourceLoc,
221 is_safepoint: bool,
222 inst: I,
223 }
224
225 /// Machine-independent lowering driver / machine-instruction container. Maintains a correspondence
226 /// from original Inst to MachInsts.
227 pub struct Lower<'func, I: VCodeInst> {
228 /// The function to lower.
229 f: &'func Function,
230
231 /// Lowered machine instructions.
232 vcode: VCodeBuilder<I>,
233
234 /// Mapping from `Value` (SSA value in IR) to virtual register.
235 value_regs: SecondaryMap<Value, ValueRegs<Reg>>,
236
237 /// Return-value vregs.
238 retval_regs: Vec<ValueRegs<Reg>>,
239
240 /// Instruction colors at block exits. From this map, we can recover all
241 /// instruction colors by scanning backward from the block end and
242 /// decrementing on any color-changing (side-effecting) instruction.
243 block_end_colors: SecondaryMap<Block, InstColor>,
244
245 /// Instruction colors at side-effecting ops. This is the *entry* color,
246 /// i.e., the version of global state that exists before an instruction
247 /// executes. For each side-effecting instruction, the *exit* color is its
248 /// entry color plus one.
249 side_effect_inst_entry_colors: FxHashMap<Inst, InstColor>,
250
251 /// Current color as we scan during lowering. While we are lowering an
252 /// instruction, this is equal to the color *at entry to* the instruction.
253 cur_scan_entry_color: Option<InstColor>,
254
255 /// Current instruction as we scan during lowering.
256 cur_inst: Option<Inst>,
257
258 /// Instruction constant values, if known.
259 inst_constants: FxHashMap<Inst, u64>,
260
261 /// Use-counts per SSA value, as counted in the input IR.
262 value_uses: SecondaryMap<Value, u32>,
263
264 /// Actual uses of each SSA value so far, incremented while lowering.
265 value_lowered_uses: SecondaryMap<Value, u32>,
266
267 /// Effectful instructions that have been sunk; they are not codegen'd at
268 /// their original locations.
269 inst_sunk: FxHashSet<Inst>,
270
271 /// Next virtual register number to allocate.
272 next_vreg: u32,
273
274 /// Insts in reverse block order, before final copy to vcode.
275 block_insts: Vec<InstTuple<I>>,
276
277 /// Ranges in `block_insts` constituting BBs.
278 block_ranges: Vec<(usize, usize)>,
279
280 /// Instructions collected for the BB in progress, in reverse order, with
281 /// source-locs attached.
282 bb_insts: Vec<InstTuple<I>>,
283
284 /// Instructions collected for the CLIF inst in progress, in forward order.
285 ir_insts: Vec<InstTuple<I>>,
286
287 /// The register to use for GetPinnedReg, if any, on this architecture.
288 pinned_reg: Option<Reg>,
289
290 /// The vreg containing the special VmContext parameter, if it is present in the current
291 /// function's signature.
292 vm_context: Option<Reg>,
293 }
294
295 /// Notion of "relocation distance". This gives an estimate of how far away a symbol will be from a
296 /// reference.
297 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
298 pub enum RelocDistance {
299 /// Target of relocation is "nearby". The threshold for this is fuzzy but should be interpreted
300 /// as approximately "within the compiled output of one module"; e.g., within AArch64's +/-
301 /// 128MB offset. If unsure, use `Far` instead.
302 Near,
303 /// Target of relocation could be anywhere in the address space.
304 Far,
305 }
306
alloc_vregs<I: VCodeInst>( ty: Type, next_vreg: &mut u32, vcode: &mut VCodeBuilder<I>, ) -> CodegenResult<ValueRegs<Reg>>307 fn alloc_vregs<I: VCodeInst>(
308 ty: Type,
309 next_vreg: &mut u32,
310 vcode: &mut VCodeBuilder<I>,
311 ) -> CodegenResult<ValueRegs<Reg>> {
312 let v = *next_vreg;
313 let (regclasses, tys) = I::rc_for_type(ty)?;
314 *next_vreg += regclasses.len() as u32;
315 let regs = match regclasses {
316 &[rc0] => ValueRegs::one(Reg::new_virtual(rc0, v)),
317 &[rc0, rc1] => ValueRegs::two(Reg::new_virtual(rc0, v), Reg::new_virtual(rc1, v + 1)),
318 #[cfg(feature = "arm32")]
319 &[rc0, rc1, rc2, rc3] => ValueRegs::four(
320 Reg::new_virtual(rc0, v),
321 Reg::new_virtual(rc1, v + 1),
322 Reg::new_virtual(rc2, v + 2),
323 Reg::new_virtual(rc3, v + 3),
324 ),
325 _ => panic!("Value must reside in 1, 2 or 4 registers"),
326 };
327 for (®_ty, ®) in tys.iter().zip(regs.regs().iter()) {
328 vcode.set_vreg_type(reg.to_virtual_reg(), reg_ty);
329 }
330 Ok(regs)
331 }
332
333 enum GenerateReturn {
334 Yes,
335 No,
336 }
337
338 impl<'func, I: VCodeInst> Lower<'func, I> {
339 /// Prepare a new lowering context for the given IR function.
new( f: &'func Function, abi: Box<dyn ABICallee<I = I>>, emit_info: I::Info, block_order: BlockLoweringOrder, ) -> CodegenResult<Lower<'func, I>>340 pub fn new(
341 f: &'func Function,
342 abi: Box<dyn ABICallee<I = I>>,
343 emit_info: I::Info,
344 block_order: BlockLoweringOrder,
345 ) -> CodegenResult<Lower<'func, I>> {
346 let constants = VCodeConstants::with_capacity(f.dfg.constants.len());
347 let mut vcode = VCodeBuilder::new(abi, emit_info, block_order, constants);
348
349 let mut next_vreg: u32 = 0;
350
351 let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid());
352
353 // Assign a vreg to each block param and each inst result.
354 for bb in f.layout.blocks() {
355 for ¶m in f.dfg.block_params(bb) {
356 let ty = f.dfg.value_type(param);
357 if value_regs[param].is_invalid() {
358 let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
359 value_regs[param] = regs;
360 log::trace!("bb {} param {}: regs {:?}", bb, param, regs);
361 }
362 }
363 for inst in f.layout.block_insts(bb) {
364 for &result in f.dfg.inst_results(inst) {
365 let ty = f.dfg.value_type(result);
366 if value_regs[result].is_invalid() {
367 let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
368 value_regs[result] = regs;
369 log::trace!(
370 "bb {} inst {} ({:?}): result regs {:?}",
371 bb,
372 inst,
373 f.dfg[inst],
374 regs,
375 );
376 }
377 }
378 }
379 }
380
381 let vm_context = vcode
382 .abi()
383 .signature()
384 .special_param_index(ArgumentPurpose::VMContext)
385 .map(|vm_context_index| {
386 let entry_block = f.layout.entry_block().unwrap();
387 let param = f.dfg.block_params(entry_block)[vm_context_index];
388 value_regs[param].only_reg().unwrap()
389 });
390
391 // Assign vreg(s) to each return value.
392 let mut retval_regs = vec![];
393 for ret in &vcode.abi().signature().returns.clone() {
394 let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?;
395 retval_regs.push(regs);
396 log::trace!("retval gets regs {:?}", regs);
397 }
398
399 // Compute instruction colors, find constant instructions, and find instructions with
400 // side-effects, in one combined pass.
401 let mut cur_color = 0;
402 let mut block_end_colors = SecondaryMap::with_default(InstColor::new(0));
403 let mut side_effect_inst_entry_colors = FxHashMap::default();
404 let mut inst_constants = FxHashMap::default();
405 let mut value_uses = SecondaryMap::with_default(0);
406 for bb in f.layout.blocks() {
407 cur_color += 1;
408 for inst in f.layout.block_insts(bb) {
409 let side_effect = has_lowering_side_effect(f, inst);
410
411 log::trace!("bb {} inst {} has color {}", bb, inst, cur_color);
412 if side_effect {
413 side_effect_inst_entry_colors.insert(inst, InstColor::new(cur_color));
414 log::trace!(" -> side-effecting; incrementing color for next inst");
415 cur_color += 1;
416 }
417
418 // Determine if this is a constant; if so, add to the table.
419 if let Some(c) = is_constant_64bit(f, inst) {
420 log::trace!(" -> constant: {}", c);
421 inst_constants.insert(inst, c);
422 }
423
424 // Count uses of all arguments.
425 for arg in f.dfg.inst_args(inst) {
426 let arg = f.dfg.resolve_aliases(*arg);
427 value_uses[arg] += 1;
428 }
429 }
430
431 block_end_colors[bb] = InstColor::new(cur_color);
432 }
433
434 Ok(Lower {
435 f,
436 vcode,
437 value_regs,
438 retval_regs,
439 block_end_colors,
440 side_effect_inst_entry_colors,
441 inst_constants,
442 next_vreg,
443 value_uses,
444 value_lowered_uses: SecondaryMap::default(),
445 inst_sunk: FxHashSet::default(),
446 cur_scan_entry_color: None,
447 cur_inst: None,
448 block_insts: vec![],
449 block_ranges: vec![],
450 bb_insts: vec![],
451 ir_insts: vec![],
452 pinned_reg: None,
453 vm_context,
454 })
455 }
456
gen_arg_setup(&mut self)457 fn gen_arg_setup(&mut self) {
458 if let Some(entry_bb) = self.f.layout.entry_block() {
459 log::trace!(
460 "gen_arg_setup: entry BB {} args are:\n{:?}",
461 entry_bb,
462 self.f.dfg.block_params(entry_bb)
463 );
464 for (i, param) in self.f.dfg.block_params(entry_bb).iter().enumerate() {
465 if !self.vcode.abi().arg_is_needed_in_body(i) {
466 continue;
467 }
468 let regs = writable_value_regs(self.value_regs[*param]);
469 for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() {
470 self.emit(insn);
471 }
472 if self.abi().signature().params[i].purpose == ArgumentPurpose::StructReturn {
473 assert!(regs.len() == 1);
474 let ty = self.abi().signature().params[i].value_type;
475 // The ABI implementation must have ensured that a StructReturn
476 // arg is present in the return values.
477 let struct_ret_idx = self
478 .abi()
479 .signature()
480 .returns
481 .iter()
482 .position(|ret| ret.purpose == ArgumentPurpose::StructReturn)
483 .expect("StructReturn return value not present!");
484 self.emit(I::gen_move(
485 Writable::from_reg(self.retval_regs[struct_ret_idx].regs()[0]),
486 regs.regs()[0].to_reg(),
487 ty,
488 ));
489 }
490 }
491 if let Some(insn) = self.vcode.abi().gen_retval_area_setup() {
492 self.emit(insn);
493 }
494 }
495 }
496
gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn)497 fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
498 // Hack: to keep `vmctx` alive, if it exists, we emit a value label here
499 // for it if debug info is requested. This ensures that it exists either
500 // in a register or spillslot throughout the entire function body, and
501 // allows for a better debugging experience.
502 if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) {
503 self.emit_value_label_marks_for_value(vmctx_val);
504 }
505
506 let retval_regs = self.retval_regs.clone();
507 for (i, regs) in retval_regs.into_iter().enumerate() {
508 let regs = writable_value_regs(regs);
509 for insn in self
510 .vcode
511 .abi()
512 .gen_copy_regs_to_retval(i, regs)
513 .into_iter()
514 {
515 self.emit(insn);
516 }
517 }
518 let inst = match gen_ret_inst {
519 GenerateReturn::Yes => self.vcode.abi().gen_ret(),
520 GenerateReturn::No => self.vcode.abi().gen_epilogue_placeholder(),
521 };
522 self.emit(inst);
523 }
524
lower_edge(&mut self, pred: Block, inst: Inst, succ: Block) -> CodegenResult<()>525 fn lower_edge(&mut self, pred: Block, inst: Inst, succ: Block) -> CodegenResult<()> {
526 log::trace!("lower_edge: pred {} succ {}", pred, succ);
527
528 let num_args = self.f.dfg.block_params(succ).len();
529 debug_assert!(num_args == self.f.dfg.inst_variable_args(inst).len());
530
531 // Most blocks have no params, so skip all the hoop-jumping below and make an early exit.
532 if num_args == 0 {
533 return Ok(());
534 }
535
536 self.cur_inst = Some(inst);
537
538 // Make up two vectors of info:
539 //
540 // * one for dsts which are to be assigned constants. We'll deal with those second, so
541 // as to minimise live ranges.
542 //
543 // * one for dsts whose sources are non-constants.
544
545 let mut const_bundles: SmallVec<[_; 16]> = SmallVec::new();
546 let mut var_bundles: SmallVec<[_; 16]> = SmallVec::new();
547
548 let mut i = 0;
549 for (dst_val, src_val) in self
550 .f
551 .dfg
552 .block_params(succ)
553 .iter()
554 .zip(self.f.dfg.inst_variable_args(inst).iter())
555 {
556 let src_val = self.f.dfg.resolve_aliases(*src_val);
557 let ty = self.f.dfg.value_type(src_val);
558
559 debug_assert!(ty == self.f.dfg.value_type(*dst_val));
560 let dst_regs = self.value_regs[*dst_val];
561
562 let input = self.get_value_as_source_or_const(src_val);
563 log::trace!("jump arg {} is {}", i, src_val);
564 i += 1;
565
566 if let Some(c) = input.constant {
567 log::trace!(" -> constant {}", c);
568 const_bundles.push((ty, writable_value_regs(dst_regs), c));
569 } else {
570 let src_regs = self.put_value_in_regs(src_val);
571 log::trace!(" -> reg {:?}", src_regs);
572 // Skip self-assignments. Not only are they pointless, they falsely trigger the
573 // overlap-check below and hence can cause a lot of unnecessary copying through
574 // temporaries.
575 if dst_regs != src_regs {
576 var_bundles.push((ty, writable_value_regs(dst_regs), src_regs));
577 }
578 }
579 }
580
581 // Deal first with the moves whose sources are variables.
582
583 // FIXME: use regalloc.rs' SparseSetU here. This would avoid all heap allocation
584 // for cases of up to circa 16 args. Currently not possible because regalloc.rs
585 // does not export it.
586 let mut src_reg_set = FxHashSet::<Reg>::default();
587 for (_, _, src_regs) in &var_bundles {
588 for ® in src_regs.regs() {
589 src_reg_set.insert(reg);
590 }
591 }
592 let mut overlaps = false;
593 'outer: for (_, dst_regs, _) in &var_bundles {
594 for ® in dst_regs.regs() {
595 if src_reg_set.contains(®.to_reg()) {
596 overlaps = true;
597 break 'outer;
598 }
599 }
600 }
601
602 // If, as is mostly the case, the source and destination register sets are non
603 // overlapping, then we can copy directly, so as to save the register allocator work.
604 if !overlaps {
605 for (ty, dst_regs, src_regs) in &var_bundles {
606 let (_, reg_tys) = I::rc_for_type(*ty)?;
607 for ((dst, src), reg_ty) in dst_regs
608 .regs()
609 .iter()
610 .zip(src_regs.regs().iter())
611 .zip(reg_tys.iter())
612 {
613 self.emit(I::gen_move(*dst, *src, *reg_ty));
614 }
615 }
616 } else {
617 // There's some overlap, so play safe and copy via temps.
618 let mut tmp_regs = SmallVec::<[ValueRegs<Writable<Reg>>; 16]>::new();
619 for (ty, _, _) in &var_bundles {
620 tmp_regs.push(self.alloc_tmp(*ty));
621 }
622 for ((ty, _, src_reg), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
623 let (_, reg_tys) = I::rc_for_type(*ty)?;
624 for ((tmp, src), reg_ty) in tmp_reg
625 .regs()
626 .iter()
627 .zip(src_reg.regs().iter())
628 .zip(reg_tys.iter())
629 {
630 self.emit(I::gen_move(*tmp, *src, *reg_ty));
631 }
632 }
633 for ((ty, dst_reg, _), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
634 let (_, reg_tys) = I::rc_for_type(*ty)?;
635 for ((dst, tmp), reg_ty) in dst_reg
636 .regs()
637 .iter()
638 .zip(tmp_reg.regs().iter())
639 .zip(reg_tys.iter())
640 {
641 self.emit(I::gen_move(*dst, tmp.to_reg(), *reg_ty));
642 }
643 }
644 }
645
646 // Now, finally, deal with the moves whose sources are constants.
647 for (ty, dst_reg, const_val) in &const_bundles {
648 for inst in I::gen_constant(*dst_reg, *const_val as u128, *ty, |ty| {
649 self.alloc_tmp(ty).only_reg().unwrap()
650 })
651 .into_iter()
652 {
653 self.emit(inst);
654 }
655 }
656
657 Ok(())
658 }
659
660 /// Has this instruction been sunk to a use-site (i.e., away from its
661 /// original location)?
is_inst_sunk(&self, inst: Inst) -> bool662 fn is_inst_sunk(&self, inst: Inst) -> bool {
663 self.inst_sunk.contains(&inst)
664 }
665
666 // Is any result of this instruction needed?
is_any_inst_result_needed(&self, inst: Inst) -> bool667 fn is_any_inst_result_needed(&self, inst: Inst) -> bool {
668 self.f
669 .dfg
670 .inst_results(inst)
671 .iter()
672 .any(|&result| self.value_lowered_uses[result] > 0)
673 }
674
lower_clif_block<B: LowerBackend<MInst = I>>( &mut self, backend: &B, block: Block, ) -> CodegenResult<()>675 fn lower_clif_block<B: LowerBackend<MInst = I>>(
676 &mut self,
677 backend: &B,
678 block: Block,
679 ) -> CodegenResult<()> {
680 self.cur_scan_entry_color = Some(self.block_end_colors[block]);
681 // Lowering loop:
682 // - For each non-branch instruction, in reverse order:
683 // - If side-effecting (load, store, branch/call/return, possible trap), or if
684 // used outside of this block, or if demanded by another inst, then lower.
685 //
686 // That's it! Lowering of side-effecting ops will force all *needed*
687 // (live) non-side-effecting ops to be lowered at the right places, via
688 // the `use_input_reg()` callback on the `LowerCtx` (that's us). That's
689 // because `use_input_reg()` sets the eager/demand bit for any insts
690 // whose result registers are used.
691 //
692 // We build up the BB in reverse instruction order in `bb_insts`.
693 // Because the machine backend calls `ctx.emit()` in forward order, we
694 // collect per-IR-inst lowered instructions in `ir_insts`, then reverse
695 // these and append to `bb_insts` as we go backward through the block.
696 // `bb_insts` are then reversed again and appended to the VCode at the
697 // end of the BB (in the toplevel driver `lower()`).
698 for inst in self.f.layout.block_insts(block).rev() {
699 let data = &self.f.dfg[inst];
700 let has_side_effect = has_lowering_side_effect(self.f, inst);
701 // If inst has been sunk to another location, skip it.
702 if self.is_inst_sunk(inst) {
703 continue;
704 }
705 // Are any outputs used at least once?
706 let value_needed = self.is_any_inst_result_needed(inst);
707 log::trace!(
708 "lower_clif_block: block {} inst {} ({:?}) is_branch {} side_effect {} value_needed {}",
709 block,
710 inst,
711 data,
712 data.opcode().is_branch(),
713 has_side_effect,
714 value_needed,
715 );
716
717 // Update scan state to color prior to this inst (as we are scanning
718 // backward).
719 self.cur_inst = Some(inst);
720 if has_side_effect {
721 let entry_color = *self
722 .side_effect_inst_entry_colors
723 .get(&inst)
724 .expect("every side-effecting inst should have a color-map entry");
725 self.cur_scan_entry_color = Some(entry_color);
726 }
727
728 // Skip lowering branches; these are handled separately
729 // (see `lower_clif_branches()` below).
730 if self.f.dfg[inst].opcode().is_branch() {
731 continue;
732 }
733
734 // Normal instruction: codegen if the instruction is side-effecting
735 // or any of its outputs its used.
736 if has_side_effect || value_needed {
737 log::trace!("lowering: inst {}: {:?}", inst, self.f.dfg[inst]);
738 backend.lower(self, inst)?;
739 // Emit value-label markers if needed, to later recover debug
740 // mappings.
741 self.emit_value_label_markers_for_inst(inst);
742 }
743 if data.opcode().is_return() {
744 // Return: handle specially, using ABI-appropriate sequence.
745 let gen_ret = if data.opcode() == Opcode::Return {
746 GenerateReturn::Yes
747 } else {
748 debug_assert!(data.opcode() == Opcode::FallthroughReturn);
749 GenerateReturn::No
750 };
751 self.gen_retval_setup(gen_ret);
752 }
753
754 let loc = self.srcloc(inst);
755 self.finish_ir_inst(loc);
756 }
757 self.cur_scan_entry_color = None;
758 Ok(())
759 }
760
get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]>761 fn get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]> {
762 if let Some(ref values_labels) = self.f.dfg.values_labels {
763 log::trace!(
764 "get_value_labels: val {} -> {} -> {:?}",
765 val,
766 self.f.dfg.resolve_aliases(val),
767 values_labels.get(&self.f.dfg.resolve_aliases(val))
768 );
769 let val = self.f.dfg.resolve_aliases(val);
770 match values_labels.get(&val) {
771 Some(&ValueLabelAssignments::Starts(ref list)) => Some(&list[..]),
772 Some(&ValueLabelAssignments::Alias { value, .. }) if depth < 10 => {
773 self.get_value_labels(value, depth + 1)
774 }
775 _ => None,
776 }
777 } else {
778 None
779 }
780 }
781
emit_value_label_marks_for_value(&mut self, val: Value)782 fn emit_value_label_marks_for_value(&mut self, val: Value) {
783 let mut markers: SmallVec<[I; 4]> = smallvec![];
784 let regs = self.value_regs[val];
785 if regs.len() > 1 {
786 return;
787 }
788 let reg = regs.only_reg().unwrap();
789
790 if let Some(label_starts) = self.get_value_labels(val, 0) {
791 let labels = label_starts
792 .iter()
793 .map(|&ValueLabelStart { label, .. }| label)
794 .collect::<FxHashSet<_>>();
795 for label in labels {
796 log::trace!(
797 "value labeling: defines val {:?} -> reg {:?} -> label {:?}",
798 val,
799 reg,
800 label,
801 );
802 markers.push(I::gen_value_label_marker(label, reg));
803 }
804 }
805 for marker in markers {
806 self.emit(marker);
807 }
808 }
809
emit_value_label_markers_for_inst(&mut self, inst: Inst)810 fn emit_value_label_markers_for_inst(&mut self, inst: Inst) {
811 if self.f.dfg.values_labels.is_none() {
812 return;
813 }
814
815 log::trace!(
816 "value labeling: srcloc {}: inst {}",
817 self.srcloc(inst),
818 inst
819 );
820 for &val in self.f.dfg.inst_results(inst) {
821 self.emit_value_label_marks_for_value(val);
822 }
823 }
824
emit_value_label_markers_for_block_args(&mut self, block: Block)825 fn emit_value_label_markers_for_block_args(&mut self, block: Block) {
826 if self.f.dfg.values_labels.is_none() {
827 return;
828 }
829
830 log::trace!("value labeling: block {}", block);
831 for &arg in self.f.dfg.block_params(block) {
832 self.emit_value_label_marks_for_value(arg);
833 }
834 self.finish_ir_inst(SourceLoc::default());
835 }
836
finish_ir_inst(&mut self, loc: SourceLoc)837 fn finish_ir_inst(&mut self, loc: SourceLoc) {
838 // `bb_insts` is kept in reverse order, so emit the instructions in
839 // reverse order.
840 for mut tuple in self.ir_insts.drain(..).rev() {
841 tuple.loc = loc;
842 self.bb_insts.push(tuple);
843 }
844 }
845
finish_bb(&mut self)846 fn finish_bb(&mut self) {
847 let start = self.block_insts.len();
848 for tuple in self.bb_insts.drain(..).rev() {
849 self.block_insts.push(tuple);
850 }
851 let end = self.block_insts.len();
852 self.block_ranges.push((start, end));
853 }
854
copy_bbs_to_vcode(&mut self)855 fn copy_bbs_to_vcode(&mut self) {
856 for &(start, end) in self.block_ranges.iter().rev() {
857 for &InstTuple {
858 loc,
859 is_safepoint,
860 ref inst,
861 } in &self.block_insts[start..end]
862 {
863 self.vcode.set_srcloc(loc);
864 self.vcode.push(inst.clone(), is_safepoint);
865 }
866 self.vcode.end_bb();
867 }
868 }
869
lower_clif_branches<B: LowerBackend<MInst = I>>( &mut self, backend: &B, block: Block, branches: &SmallVec<[Inst; 2]>, targets: &SmallVec<[MachLabel; 2]>, ) -> CodegenResult<()>870 fn lower_clif_branches<B: LowerBackend<MInst = I>>(
871 &mut self,
872 backend: &B,
873 block: Block,
874 branches: &SmallVec<[Inst; 2]>,
875 targets: &SmallVec<[MachLabel; 2]>,
876 ) -> CodegenResult<()> {
877 log::trace!(
878 "lower_clif_branches: block {} branches {:?} targets {:?}",
879 block,
880 branches,
881 targets,
882 );
883 // When considering code-motion opportunities, consider the current
884 // program point to be the first branch.
885 self.cur_inst = Some(branches[0]);
886 backend.lower_branch_group(self, branches, targets)?;
887 let loc = self.srcloc(branches[0]);
888 self.finish_ir_inst(loc);
889 Ok(())
890 }
891
collect_branches_and_targets( &self, bindex: BlockIndex, _bb: Block, branches: &mut SmallVec<[Inst; 2]>, targets: &mut SmallVec<[MachLabel; 2]>, )892 fn collect_branches_and_targets(
893 &self,
894 bindex: BlockIndex,
895 _bb: Block,
896 branches: &mut SmallVec<[Inst; 2]>,
897 targets: &mut SmallVec<[MachLabel; 2]>,
898 ) {
899 branches.clear();
900 targets.clear();
901 let mut last_inst = None;
902 for &(inst, succ) in self.vcode.block_order().succ_indices(bindex) {
903 // Avoid duplicates: this ensures a br_table is only inserted once.
904 if last_inst != Some(inst) {
905 branches.push(inst);
906 } else {
907 debug_assert!(self.f.dfg[inst].opcode() == Opcode::BrTable);
908 debug_assert!(branches.len() == 1);
909 }
910 last_inst = Some(inst);
911 targets.push(MachLabel::from_block(succ));
912 }
913 }
914
915 /// Lower the function.
lower<B: LowerBackend<MInst = I>>( mut self, backend: &B, ) -> CodegenResult<(VCode<I>, StackmapRequestInfo)>916 pub fn lower<B: LowerBackend<MInst = I>>(
917 mut self,
918 backend: &B,
919 ) -> CodegenResult<(VCode<I>, StackmapRequestInfo)> {
920 log::trace!("about to lower function: {:?}", self.f);
921
922 // Initialize the ABI object, giving it a temp if requested.
923 let maybe_tmp = if let Some(temp_ty) = self.vcode.abi().temp_needed() {
924 Some(self.alloc_tmp(temp_ty).only_reg().unwrap())
925 } else {
926 None
927 };
928 self.vcode.abi().init(maybe_tmp);
929
930 // Get the pinned reg here (we only parameterize this function on `B`,
931 // not the whole `Lower` impl).
932 self.pinned_reg = backend.maybe_pinned_reg();
933
934 self.vcode.set_entry(0);
935
936 // Reused vectors for branch lowering.
937 let mut branches: SmallVec<[Inst; 2]> = SmallVec::new();
938 let mut targets: SmallVec<[MachLabel; 2]> = SmallVec::new();
939
940 // get a copy of the lowered order; we hold this separately because we
941 // need a mut ref to the vcode to mutate it below.
942 let lowered_order: SmallVec<[LoweredBlock; 64]> = self
943 .vcode
944 .block_order()
945 .lowered_order()
946 .iter()
947 .cloned()
948 .collect();
949
950 // Main lowering loop over lowered blocks.
951 for (bindex, lb) in lowered_order.iter().enumerate().rev() {
952 let bindex = bindex as BlockIndex;
953
954 // Lower the block body in reverse order (see comment in
955 // `lower_clif_block()` for rationale).
956
957 // End branches.
958 if let Some(bb) = lb.orig_block() {
959 self.collect_branches_and_targets(bindex, bb, &mut branches, &mut targets);
960 if branches.len() > 0 {
961 self.lower_clif_branches(backend, bb, &branches, &targets)?;
962 self.finish_ir_inst(self.srcloc(branches[0]));
963 }
964 } else {
965 // If no orig block, this must be a pure edge block; get the successor and
966 // emit a jump.
967 let (_, succ) = self.vcode.block_order().succ_indices(bindex)[0];
968 self.emit(I::gen_jump(MachLabel::from_block(succ)));
969 self.finish_ir_inst(SourceLoc::default());
970 }
971
972 // Out-edge phi moves.
973 if let Some((pred, inst, succ)) = lb.out_edge() {
974 self.lower_edge(pred, inst, succ)?;
975 self.finish_ir_inst(SourceLoc::default());
976 }
977 // Original block body.
978 if let Some(bb) = lb.orig_block() {
979 self.lower_clif_block(backend, bb)?;
980 self.emit_value_label_markers_for_block_args(bb);
981 }
982 // In-edge phi moves.
983 if let Some((pred, inst, succ)) = lb.in_edge() {
984 self.lower_edge(pred, inst, succ)?;
985 self.finish_ir_inst(SourceLoc::default());
986 }
987
988 if bindex == 0 {
989 // Set up the function with arg vreg inits.
990 self.gen_arg_setup();
991 self.finish_ir_inst(SourceLoc::default());
992 }
993
994 self.finish_bb();
995 }
996
997 self.copy_bbs_to_vcode();
998
999 // Now that we've emitted all instructions into the VCodeBuilder, let's build the VCode.
1000 let (vcode, stack_map_info) = self.vcode.build();
1001 log::trace!("built vcode: {:?}", vcode);
1002
1003 Ok((vcode, stack_map_info))
1004 }
1005
put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg>1006 fn put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg> {
1007 log::trace!("put_value_in_reg: val {}", val);
1008 let mut regs = self.value_regs[val];
1009 log::trace!(" -> regs {:?}", regs);
1010 assert!(regs.is_valid());
1011
1012 self.value_lowered_uses[val] += 1;
1013
1014 // Pinned-reg hack: if backend specifies a fixed pinned register, use it
1015 // directly when we encounter a GetPinnedReg op, rather than lowering
1016 // the actual op, and do not return the source inst to the caller; the
1017 // value comes "out of the ether" and we will not force generation of
1018 // the superfluous move.
1019 if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) {
1020 if self.f.dfg[i].opcode() == Opcode::GetPinnedReg {
1021 if let Some(pr) = self.pinned_reg {
1022 regs = ValueRegs::one(pr);
1023 }
1024 }
1025 }
1026
1027 regs
1028 }
1029
1030 /// Get the actual inputs for a value. This is the implementation for
1031 /// `get_input()` but starting from the SSA value, which is not exposed to
1032 /// the backend.
get_value_as_source_or_const(&self, val: Value) -> NonRegInput1033 fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput {
1034 log::trace!(
1035 "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}",
1036 val,
1037 self.cur_inst,
1038 self.cur_scan_entry_color,
1039 );
1040 let inst = match self.f.dfg.value_def(val) {
1041 // OK to merge source instruction if (i) we have a source
1042 // instruction, and:
1043 // - It has no side-effects, OR
1044 // - It has a side-effect, has one output value, that one output has
1045 // only one use (this one), and the instruction's color is *one less
1046 // than* the current scan color.
1047 //
1048 // This latter set of conditions is testing whether a
1049 // side-effecting instruction can sink to the current scan
1050 // location; this is possible if the in-color of this inst is
1051 // equal to the out-color of the producing inst, so no other
1052 // side-effecting ops occur between them (which will only be true
1053 // if they are in the same BB, because color increments at each BB
1054 // start).
1055 //
1056 // If it is actually sunk, then in `merge_inst()`, we update the
1057 // scan color so that as we scan over the range past which the
1058 // instruction was sunk, we allow other instructions (that came
1059 // prior to the sunk instruction) to sink.
1060 ValueDef::Result(src_inst, result_idx) => {
1061 let src_side_effect = has_lowering_side_effect(self.f, src_inst);
1062 log::trace!(" -> src inst {}", src_inst);
1063 log::trace!(" -> has lowering side effect: {}", src_side_effect);
1064 if !src_side_effect {
1065 // Pure instruction: always possible to sink.
1066 Some((src_inst, result_idx))
1067 } else {
1068 // Side-effect: test whether this is the only use of the
1069 // only result of the instruction, and whether colors allow
1070 // the code-motion.
1071 if self.cur_scan_entry_color.is_some()
1072 && self.value_uses[val] == 1
1073 && self.value_lowered_uses[val] == 0
1074 && self.num_outputs(src_inst) == 1
1075 && self
1076 .side_effect_inst_entry_colors
1077 .get(&src_inst)
1078 .unwrap()
1079 .get()
1080 + 1
1081 == self.cur_scan_entry_color.unwrap().get()
1082 {
1083 Some((src_inst, 0))
1084 } else {
1085 None
1086 }
1087 }
1088 }
1089 _ => None,
1090 };
1091 let constant = inst.and_then(|(inst, _)| self.get_constant(inst));
1092
1093 NonRegInput { inst, constant }
1094 }
1095 }
1096
1097 impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
1098 type I = I;
1099
abi(&mut self) -> &mut dyn ABICallee<I = I>1100 fn abi(&mut self) -> &mut dyn ABICallee<I = I> {
1101 self.vcode.abi()
1102 }
1103
retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>1104 fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>> {
1105 writable_value_regs(self.retval_regs[idx])
1106 }
1107
get_vm_context(&self) -> Option<Reg>1108 fn get_vm_context(&self) -> Option<Reg> {
1109 self.vm_context
1110 }
1111
data(&self, ir_inst: Inst) -> &InstructionData1112 fn data(&self, ir_inst: Inst) -> &InstructionData {
1113 &self.f.dfg[ir_inst]
1114 }
1115
ty(&self, ir_inst: Inst) -> Type1116 fn ty(&self, ir_inst: Inst) -> Type {
1117 self.f.dfg.ctrl_typevar(ir_inst)
1118 }
1119
call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>1120 fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> {
1121 match &self.f.dfg[ir_inst] {
1122 &InstructionData::Call { func_ref, .. }
1123 | &InstructionData::FuncAddr { func_ref, .. } => {
1124 let funcdata = &self.f.dfg.ext_funcs[func_ref];
1125 let dist = funcdata.reloc_distance();
1126 Some((&funcdata.name, dist))
1127 }
1128 _ => None,
1129 }
1130 }
1131
call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>1132 fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> {
1133 match &self.f.dfg[ir_inst] {
1134 &InstructionData::Call { func_ref, .. } => {
1135 let funcdata = &self.f.dfg.ext_funcs[func_ref];
1136 Some(&self.f.dfg.signatures[funcdata.signature])
1137 }
1138 &InstructionData::CallIndirect { sig_ref, .. } => Some(&self.f.dfg.signatures[sig_ref]),
1139 _ => None,
1140 }
1141 }
1142
symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>1143 fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)> {
1144 match &self.f.dfg[ir_inst] {
1145 &InstructionData::UnaryGlobalValue { global_value, .. } => {
1146 let gvdata = &self.f.global_values[global_value];
1147 match gvdata {
1148 &GlobalValueData::Symbol {
1149 ref name,
1150 ref offset,
1151 ..
1152 } => {
1153 let offset = offset.bits();
1154 let dist = gvdata.maybe_reloc_distance().unwrap();
1155 Some((name, dist, offset))
1156 }
1157 _ => None,
1158 }
1159 }
1160 _ => None,
1161 }
1162 }
1163
memflags(&self, ir_inst: Inst) -> Option<MemFlags>1164 fn memflags(&self, ir_inst: Inst) -> Option<MemFlags> {
1165 match &self.f.dfg[ir_inst] {
1166 &InstructionData::AtomicCas { flags, .. } => Some(flags),
1167 &InstructionData::AtomicRmw { flags, .. } => Some(flags),
1168 &InstructionData::Load { flags, .. }
1169 | &InstructionData::LoadComplex { flags, .. }
1170 | &InstructionData::LoadNoOffset { flags, .. }
1171 | &InstructionData::Store { flags, .. }
1172 | &InstructionData::StoreComplex { flags, .. } => Some(flags),
1173 &InstructionData::StoreNoOffset { flags, .. } => Some(flags),
1174 _ => None,
1175 }
1176 }
1177
srcloc(&self, ir_inst: Inst) -> SourceLoc1178 fn srcloc(&self, ir_inst: Inst) -> SourceLoc {
1179 self.f.srclocs[ir_inst]
1180 }
1181
num_inputs(&self, ir_inst: Inst) -> usize1182 fn num_inputs(&self, ir_inst: Inst) -> usize {
1183 self.f.dfg.inst_args(ir_inst).len()
1184 }
1185
num_outputs(&self, ir_inst: Inst) -> usize1186 fn num_outputs(&self, ir_inst: Inst) -> usize {
1187 self.f.dfg.inst_results(ir_inst).len()
1188 }
1189
input_ty(&self, ir_inst: Inst, idx: usize) -> Type1190 fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type {
1191 let val = self.f.dfg.inst_args(ir_inst)[idx];
1192 let val = self.f.dfg.resolve_aliases(val);
1193 self.f.dfg.value_type(val)
1194 }
1195
output_ty(&self, ir_inst: Inst, idx: usize) -> Type1196 fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type {
1197 self.f.dfg.value_type(self.f.dfg.inst_results(ir_inst)[idx])
1198 }
1199
get_constant(&self, ir_inst: Inst) -> Option<u64>1200 fn get_constant(&self, ir_inst: Inst) -> Option<u64> {
1201 self.inst_constants.get(&ir_inst).cloned()
1202 }
1203
get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput1204 fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput {
1205 let val = self.f.dfg.inst_args(ir_inst)[idx];
1206 let val = self.f.dfg.resolve_aliases(val);
1207 self.get_value_as_source_or_const(val)
1208 }
1209
put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>1210 fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg> {
1211 let val = self.f.dfg.inst_args(ir_inst)[idx];
1212 let val = self.f.dfg.resolve_aliases(val);
1213 self.put_value_in_regs(val)
1214 }
1215
get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>1216 fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>> {
1217 let val = self.f.dfg.inst_results(ir_inst)[idx];
1218 writable_value_regs(self.value_regs[val])
1219 }
1220
alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>1221 fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>> {
1222 writable_value_regs(alloc_vregs(ty, &mut self.next_vreg, &mut self.vcode).unwrap())
1223 }
1224
emit(&mut self, mach_inst: I)1225 fn emit(&mut self, mach_inst: I) {
1226 self.ir_insts.push(InstTuple {
1227 loc: SourceLoc::default(),
1228 is_safepoint: false,
1229 inst: mach_inst,
1230 });
1231 }
1232
emit_safepoint(&mut self, mach_inst: I)1233 fn emit_safepoint(&mut self, mach_inst: I) {
1234 self.ir_insts.push(InstTuple {
1235 loc: SourceLoc::default(),
1236 is_safepoint: true,
1237 inst: mach_inst,
1238 });
1239 }
1240
sink_inst(&mut self, ir_inst: Inst)1241 fn sink_inst(&mut self, ir_inst: Inst) {
1242 assert!(has_lowering_side_effect(self.f, ir_inst));
1243 assert!(self.cur_scan_entry_color.is_some());
1244
1245 let sunk_inst_entry_color = self
1246 .side_effect_inst_entry_colors
1247 .get(&ir_inst)
1248 .cloned()
1249 .unwrap();
1250 let sunk_inst_exit_color = InstColor::new(sunk_inst_entry_color.get() + 1);
1251 assert!(sunk_inst_exit_color == self.cur_scan_entry_color.unwrap());
1252 self.cur_scan_entry_color = Some(sunk_inst_entry_color);
1253 self.inst_sunk.insert(ir_inst);
1254 }
1255
get_constant_data(&self, constant_handle: Constant) -> &ConstantData1256 fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData {
1257 self.f.dfg.constants.get(constant_handle)
1258 }
1259
use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant1260 fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant {
1261 self.vcode.constants().insert(constant)
1262 }
1263
get_immediate(&self, ir_inst: Inst) -> Option<DataValue>1264 fn get_immediate(&self, ir_inst: Inst) -> Option<DataValue> {
1265 let inst_data = self.data(ir_inst);
1266 match inst_data {
1267 InstructionData::Shuffle { mask, .. } => {
1268 let buffer = self.f.dfg.immediates.get(mask.clone()).unwrap().as_slice();
1269 let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer"));
1270 Some(value)
1271 }
1272 InstructionData::UnaryConst {
1273 constant_handle, ..
1274 } => {
1275 let buffer = self.f.dfg.constants.get(constant_handle.clone()).as_slice();
1276 let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer"));
1277 Some(value)
1278 }
1279 _ => inst_data.imm_value(),
1280 }
1281 }
1282
ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg1283 fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg {
1284 if reg.is_virtual() {
1285 reg
1286 } else {
1287 let new_reg = self.alloc_tmp(ty).only_reg().unwrap();
1288 self.emit(I::gen_move(new_reg, reg, ty));
1289 new_reg.to_reg()
1290 }
1291 }
1292 }
1293
1294 /// Visit all successors of a block with a given visitor closure.
visit_block_succs<F: FnMut(Inst, Block)>(f: &Function, block: Block, mut visit: F)1295 pub(crate) fn visit_block_succs<F: FnMut(Inst, Block)>(f: &Function, block: Block, mut visit: F) {
1296 for inst in f.layout.block_likely_branches(block) {
1297 if f.dfg[inst].opcode().is_branch() {
1298 visit_branch_targets(f, block, inst, &mut visit);
1299 }
1300 }
1301 }
1302
visit_branch_targets<F: FnMut(Inst, Block)>( f: &Function, block: Block, inst: Inst, visit: &mut F, )1303 fn visit_branch_targets<F: FnMut(Inst, Block)>(
1304 f: &Function,
1305 block: Block,
1306 inst: Inst,
1307 visit: &mut F,
1308 ) {
1309 if f.dfg[inst].opcode() == Opcode::Fallthrough {
1310 visit(inst, f.layout.next_block(block).unwrap());
1311 } else {
1312 match f.dfg[inst].analyze_branch(&f.dfg.value_lists) {
1313 BranchInfo::NotABranch => {}
1314 BranchInfo::SingleDest(dest, _) => {
1315 visit(inst, dest);
1316 }
1317 BranchInfo::Table(table, maybe_dest) => {
1318 if let Some(dest) = maybe_dest {
1319 visit(inst, dest);
1320 }
1321 for &dest in f.jump_tables[table].as_slice() {
1322 visit(inst, dest);
1323 }
1324 }
1325 }
1326 }
1327 }
1328