1 //! Implementation of the standard AArch64 ABI.
2
3 use crate::ir;
4 use crate::ir::types;
5 use crate::ir::types::*;
6 use crate::ir::{ArgumentExtension, StackSlot};
7 use crate::isa;
8 use crate::isa::aarch64::{self, inst::*};
9 use crate::machinst::*;
10 use crate::settings;
11
12 use alloc::vec::Vec;
13
14 use regalloc::{RealReg, Reg, RegClass, Set, SpillSlot, Writable};
15
16 use log::debug;
17
18 /// A location for an argument or return value.
19 #[derive(Clone, Copy, Debug)]
20 enum ABIArg {
21 /// In a real register.
22 Reg(RealReg, ir::Type),
23 /// Arguments only: on stack, at given offset from SP at entry.
24 Stack(i64, ir::Type),
25 }
26
27 /// AArch64 ABI information shared between body (callee) and caller.
28 struct ABISig {
29 args: Vec<ABIArg>,
30 rets: Vec<ABIArg>,
31 stack_arg_space: i64,
32 call_conv: isa::CallConv,
33 }
34
35 // Spidermonkey specific ABI convention.
36
37 /// This is SpiderMonkey's `WasmTableCallSigReg`.
38 static BALDRDASH_SIG_REG: u8 = 10;
39
40 /// This is SpiderMonkey's `WasmTlsReg`.
41 static BALDRDASH_TLS_REG: u8 = 23;
42
43 // These two lists represent the registers the JIT may *not* use at any point in generated code.
44 //
45 // So these are callee-preserved from the JIT's point of view, and every register not in this list
46 // has to be caller-preserved by definition.
47 //
48 // Keep these lists in sync with the NonAllocatableMask set in Spidermonkey's
49 // Architecture-arm64.cpp.
50
51 // Indexed by physical register number.
52 #[rustfmt::skip]
53 static BALDRDASH_JIT_CALLEE_SAVED_GPR: &[bool] = &[
54 /* 0 = */ false, false, false, false, false, false, false, false,
55 /* 8 = */ false, false, false, false, false, false, false, false,
56 /* 16 = */ true /* x16 / ip1 */, true /* x17 / ip2 */, true /* x18 / TLS */, false,
57 /* 20 = */ false, false, false, false,
58 /* 24 = */ false, false, false, false,
59 // There should be 28, the pseudo stack pointer in this list, however the wasm stubs trash it
60 // gladly right now.
61 /* 28 = */ false, false, true /* x30 = FP */, false /* x31 = SP */
62 ];
63
64 #[rustfmt::skip]
65 static BALDRDASH_JIT_CALLEE_SAVED_FPU: &[bool] = &[
66 /* 0 = */ false, false, false, false, false, false, false, false,
67 /* 8 = */ false, false, false, false, false, false, false, false,
68 /* 16 = */ false, false, false, false, false, false, false, false,
69 /* 24 = */ false, false, false, false, false, false, false, true /* v31 / d31 */
70 ];
71
72 /// Try to fill a Baldrdash register, returning it if it was found.
try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Option<ABIArg>73 fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Option<ABIArg> {
74 if call_conv.extends_baldrdash() {
75 match ¶m.purpose {
76 &ir::ArgumentPurpose::VMContext => {
77 // This is SpiderMonkey's `WasmTlsReg`.
78 Some(ABIArg::Reg(
79 xreg(BALDRDASH_TLS_REG).to_real_reg(),
80 ir::types::I64,
81 ))
82 }
83 &ir::ArgumentPurpose::SignatureId => {
84 // This is SpiderMonkey's `WasmTableCallSigReg`.
85 Some(ABIArg::Reg(
86 xreg(BALDRDASH_SIG_REG).to_real_reg(),
87 ir::types::I64,
88 ))
89 }
90 _ => None,
91 }
92 } else {
93 None
94 }
95 }
96
97 /// Process a list of parameters or return values and allocate them to X-regs,
98 /// V-regs, and stack slots.
99 ///
100 /// Returns the list of argument locations, and the stack-space used (rounded up
101 /// to a 16-byte-aligned boundary).
compute_arg_locs(call_conv: isa::CallConv, params: &[ir::AbiParam]) -> (Vec<ABIArg>, i64)102 fn compute_arg_locs(call_conv: isa::CallConv, params: &[ir::AbiParam]) -> (Vec<ABIArg>, i64) {
103 // See AArch64 ABI (https://c9x.me/compile/bib/abi-arm64.pdf), sections 5.4.
104 let mut next_xreg = 0;
105 let mut next_vreg = 0;
106 let mut next_stack: u64 = 0;
107 let mut ret = vec![];
108
109 for param in params {
110 // Validate "purpose".
111 match ¶m.purpose {
112 &ir::ArgumentPurpose::VMContext
113 | &ir::ArgumentPurpose::Normal
114 | &ir::ArgumentPurpose::StackLimit
115 | &ir::ArgumentPurpose::SignatureId => {}
116 _ => panic!(
117 "Unsupported argument purpose {:?} in signature: {:?}",
118 param.purpose, params
119 ),
120 }
121
122 if in_int_reg(param.value_type) {
123 if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
124 ret.push(param);
125 } else if next_xreg < 8 {
126 ret.push(ABIArg::Reg(xreg(next_xreg).to_real_reg(), param.value_type));
127 next_xreg += 1;
128 } else {
129 ret.push(ABIArg::Stack(next_stack as i64, param.value_type));
130 next_stack += 8;
131 }
132 } else if in_vec_reg(param.value_type) {
133 if next_vreg < 8 {
134 ret.push(ABIArg::Reg(vreg(next_vreg).to_real_reg(), param.value_type));
135 next_vreg += 1;
136 } else {
137 let size: u64 = match param.value_type {
138 F32 | F64 => 8,
139 _ => panic!("Unsupported vector-reg argument type"),
140 };
141 // Align.
142 debug_assert!(size.is_power_of_two());
143 next_stack = (next_stack + size - 1) & !(size - 1);
144 ret.push(ABIArg::Stack(next_stack as i64, param.value_type));
145 next_stack += size;
146 }
147 }
148 }
149
150 next_stack = (next_stack + 15) & !15;
151
152 (ret, next_stack as i64)
153 }
154
155 impl ABISig {
from_func_sig(sig: &ir::Signature) -> ABISig156 fn from_func_sig(sig: &ir::Signature) -> ABISig {
157 // Compute args and retvals from signature.
158 // TODO: pass in arg-mode or ret-mode. (Does not matter
159 // for the types of arguments/return values that we support.)
160 let (args, stack_arg_space) = compute_arg_locs(sig.call_conv, &sig.params);
161 let (rets, _) = compute_arg_locs(sig.call_conv, &sig.returns);
162
163 // Verify that there are no return values on the stack.
164 debug_assert!(rets.iter().all(|a| match a {
165 &ABIArg::Stack(..) => false,
166 _ => true,
167 }));
168
169 ABISig {
170 args,
171 rets,
172 stack_arg_space,
173 call_conv: sig.call_conv,
174 }
175 }
176 }
177
178 /// AArch64 ABI object for a function body.
179 pub struct AArch64ABIBody {
180 /// Signature: arg and retval regs.
181 sig: ABISig,
182 /// Offsets to each stackslot.
183 stackslots: Vec<u32>,
184 /// Total stack size of all stackslots.
185 stackslots_size: u32,
186 /// Clobbered registers, from regalloc.
187 clobbered: Set<Writable<RealReg>>,
188 /// Total number of spillslots, from regalloc.
189 spillslots: Option<usize>,
190 /// Total frame size.
191 frame_size: Option<u32>,
192 /// Calling convention this function expects.
193 call_conv: isa::CallConv,
194 /// The settings controlling this function's compilation.
195 flags: settings::Flags,
196 /// Whether or not this function is a "leaf", meaning it calls no other
197 /// functions
198 is_leaf: bool,
199 /// If this function has a stack limit specified, then `Reg` is where the
200 /// stack limit will be located after the instructions specified have been
201 /// executed.
202 ///
203 /// Note that this is intended for insertion into the prologue, if
204 /// present. Also note that because the instructions here execute in the
205 /// prologue this happens after legalization/register allocation/etc so we
206 /// need to be extremely careful with each instruction. The instructions are
207 /// manually register-allocated and carefully only use caller-saved
208 /// registers and keep nothing live after this sequence of instructions.
209 stack_limit: Option<(Reg, Vec<Inst>)>,
210 }
211
in_int_reg(ty: ir::Type) -> bool212 fn in_int_reg(ty: ir::Type) -> bool {
213 match ty {
214 types::I8 | types::I16 | types::I32 | types::I64 => true,
215 types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
216 _ => false,
217 }
218 }
219
in_vec_reg(ty: ir::Type) -> bool220 fn in_vec_reg(ty: ir::Type) -> bool {
221 match ty {
222 types::F32 | types::F64 => true,
223 _ => false,
224 }
225 }
226
227 /// Generates the instructions necessary for the `gv` to be materialized into a
228 /// register.
229 ///
230 /// This function will return a register that will contain the result of
231 /// evaluating `gv`. It will also return any instructions necessary to calculate
232 /// the value of the register.
233 ///
234 /// Note that global values are typically lowered to instructions via the
235 /// standard legalization pass. Unfortunately though prologue generation happens
236 /// so late in the pipeline that we can't use these legalization passes to
237 /// generate the instructions for `gv`. As a result we duplicate some lowering
238 /// of `gv` here and support only some global values. This is similar to what
239 /// the x86 backend does for now, and hopefully this can be somewhat cleaned up
240 /// in the future too!
241 ///
242 /// Also note that this function will make use of `writable_spilltmp_reg()` as a
243 /// temporary register to store values in if necessary. Currently after we write
244 /// to this register there's guaranteed to be no spilled values between where
245 /// it's used, because we're not participating in register allocation anyway!
gen_stack_limit(f: &ir::Function, abi: &ABISig, gv: ir::GlobalValue) -> (Reg, Vec<Inst>)246 fn gen_stack_limit(f: &ir::Function, abi: &ABISig, gv: ir::GlobalValue) -> (Reg, Vec<Inst>) {
247 let mut insts = Vec::new();
248 let reg = generate_gv(f, abi, gv, &mut insts);
249 return (reg, insts);
250
251 fn generate_gv(
252 f: &ir::Function,
253 abi: &ABISig,
254 gv: ir::GlobalValue,
255 insts: &mut Vec<Inst>,
256 ) -> Reg {
257 match f.global_values[gv] {
258 // Return the direct register the vmcontext is in
259 ir::GlobalValueData::VMContext => {
260 get_special_purpose_param_register(f, abi, ir::ArgumentPurpose::VMContext)
261 .expect("no vmcontext parameter found")
262 }
263 // Load our base value into a register, then load from that register
264 // in to a temporary register.
265 ir::GlobalValueData::Load {
266 base,
267 offset,
268 global_type: _,
269 readonly: _,
270 } => {
271 let base = generate_gv(f, abi, base, insts);
272 let into_reg = writable_spilltmp_reg();
273 let mem = if let Some(offset) =
274 UImm12Scaled::maybe_from_i64(offset.into(), ir::types::I8)
275 {
276 MemArg::UnsignedOffset(base, offset)
277 } else {
278 let offset: i64 = offset.into();
279 insts.extend(Inst::load_constant(into_reg, offset as u64));
280 MemArg::RegReg(base, into_reg.to_reg())
281 };
282 insts.push(Inst::ULoad64 {
283 rd: into_reg,
284 mem,
285 srcloc: None,
286 });
287 return into_reg.to_reg();
288 }
289 ref other => panic!("global value for stack limit not supported: {}", other),
290 }
291 }
292 }
293
get_special_purpose_param_register( f: &ir::Function, abi: &ABISig, purpose: ir::ArgumentPurpose, ) -> Option<Reg>294 fn get_special_purpose_param_register(
295 f: &ir::Function,
296 abi: &ABISig,
297 purpose: ir::ArgumentPurpose,
298 ) -> Option<Reg> {
299 let idx = f.signature.special_param_index(purpose)?;
300 match abi.args[idx] {
301 ABIArg::Reg(reg, _) => Some(reg.to_reg()),
302 ABIArg::Stack(..) => None,
303 }
304 }
305
306 impl AArch64ABIBody {
307 /// Create a new body ABI instance.
new(f: &ir::Function, flags: settings::Flags) -> Self308 pub fn new(f: &ir::Function, flags: settings::Flags) -> Self {
309 debug!("AArch64 ABI: func signature {:?}", f.signature);
310
311 let sig = ABISig::from_func_sig(&f.signature);
312
313 let call_conv = f.signature.call_conv;
314 // Only these calling conventions are supported.
315 debug_assert!(
316 call_conv == isa::CallConv::SystemV
317 || call_conv == isa::CallConv::Fast
318 || call_conv == isa::CallConv::Cold
319 || call_conv.extends_baldrdash(),
320 "Unsupported calling convention: {:?}",
321 call_conv
322 );
323
324 // Compute stackslot locations and total stackslot size.
325 let mut stack_offset: u32 = 0;
326 let mut stackslots = vec![];
327 for (stackslot, data) in f.stack_slots.iter() {
328 let off = stack_offset;
329 stack_offset += data.size;
330 stack_offset = (stack_offset + 7) & !7;
331 debug_assert_eq!(stackslot.as_u32() as usize, stackslots.len());
332 stackslots.push(off);
333 }
334
335 // Figure out what instructions, if any, will be needed to check the
336 // stack limit. This can either be specified as a special-purpose
337 // argument or as a global value which often calculates the stack limit
338 // from the arguments.
339 let stack_limit =
340 get_special_purpose_param_register(f, &sig, ir::ArgumentPurpose::StackLimit)
341 .map(|reg| (reg, Vec::new()))
342 .or_else(|| f.stack_limit.map(|gv| gen_stack_limit(f, &sig, gv)));
343
344 Self {
345 sig,
346 stackslots,
347 stackslots_size: stack_offset,
348 clobbered: Set::empty(),
349 spillslots: None,
350 frame_size: None,
351 call_conv,
352 flags,
353 is_leaf: f.is_leaf(),
354 stack_limit,
355 }
356 }
357
358 /// Returns the size of a function call frame (including return address and FP) for this
359 /// function's body.
frame_size(&self) -> i64360 fn frame_size(&self) -> i64 {
361 if self.call_conv.extends_baldrdash() {
362 let num_words = self.flags.baldrdash_prologue_words() as i64;
363 debug_assert!(num_words > 0, "baldrdash must set baldrdash_prologue_words");
364 debug_assert_eq!(num_words % 2, 0, "stack must be 16-aligned");
365 num_words * 8
366 } else {
367 16 // frame pointer + return address.
368 }
369 }
370
371 /// Inserts instructions necessary for checking the stack limit into the
372 /// prologue.
373 ///
374 /// This function will generate instructions necessary for perform a stack
375 /// check at the header of a function. The stack check is intended to trap
376 /// if the stack pointer goes below a particular threshold, preventing stack
377 /// overflow in wasm or other code. The `stack_limit` argument here is the
378 /// register which holds the threshold below which we're supposed to trap.
379 /// This function is known to allocate `stack_size` bytes and we'll push
380 /// instructions onto `insts`.
381 ///
382 /// Note that the instructions generated here are special because this is
383 /// happening so late in the pipeline (e.g. after register allocation). This
384 /// means that we need to do manual register allocation here and also be
385 /// careful to not clobber any callee-saved or argument registers. For now
386 /// this routine makes do with the `writable_spilltmp_reg` as one temporary
387 /// register, and a second register of `x16` which is caller-saved. This
388 /// should be fine for us since no spills should happen in this sequence of
389 /// instructions, so our register won't get accidentally clobbered.
390 ///
391 /// No values can be live after the prologue, but in this case that's ok
392 /// because we just need to perform a stack check before progressing with
393 /// the rest of the function.
insert_stack_check(&self, stack_limit: Reg, stack_size: u32, insts: &mut Vec<Inst>)394 fn insert_stack_check(&self, stack_limit: Reg, stack_size: u32, insts: &mut Vec<Inst>) {
395 // With no explicit stack allocated we can just emit the simple check of
396 // the stack registers against the stack limit register, and trap if
397 // it's out of bounds.
398 if stack_size == 0 {
399 return push_check(stack_limit, insts);
400 }
401
402 // Note that the 32k stack size here is pretty special. See the
403 // documentation in x86/abi.rs for why this is here. The general idea is
404 // that we're protecting against overflow in the addition that happens
405 // below.
406 if stack_size >= 32 * 1024 {
407 push_check(stack_limit, insts);
408 }
409
410 // Add the `stack_size` to `stack_limit`, placing the result in
411 // `scratch`.
412 //
413 // Note though that `stack_limit`'s register may be the same as
414 // `scratch`. If our stack size doesn't fit into an immediate this
415 // means we need a second scratch register for loading the stack size
416 // into a register. We use `x16` here since it's caller-saved and we're
417 // in the function prologue and nothing else is allocated to it yet.
418 let scratch = writable_spilltmp_reg();
419 let stack_size = u64::from(stack_size);
420 if let Some(imm12) = Imm12::maybe_from_u64(stack_size) {
421 insts.push(Inst::AluRRImm12 {
422 alu_op: ALUOp::Add64,
423 rd: scratch,
424 rn: stack_limit,
425 imm12,
426 });
427 } else {
428 let scratch2 = 16;
429 insts.extend(Inst::load_constant(
430 Writable::from_reg(xreg(scratch2)),
431 stack_size.into(),
432 ));
433 insts.push(Inst::AluRRRExtend {
434 alu_op: ALUOp::Add64,
435 rd: scratch,
436 rn: stack_limit,
437 rm: xreg(scratch2),
438 extendop: ExtendOp::UXTX,
439 });
440 }
441 push_check(scratch.to_reg(), insts);
442
443 fn push_check(stack_limit: Reg, insts: &mut Vec<Inst>) {
444 insts.push(Inst::AluRRR {
445 alu_op: ALUOp::SubS64XR,
446 rd: writable_zero_reg(),
447 rn: stack_reg(),
448 rm: stack_limit,
449 });
450 insts.push(Inst::CondBrLowered {
451 target: BranchTarget::ResolvedOffset(8),
452 // Here `Hs` == "higher or same" when interpreting the two
453 // operands as unsigned integers.
454 kind: CondBrKind::Cond(Cond::Hs),
455 });
456 insts.push(Inst::Udf {
457 trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow),
458 });
459 }
460 }
461 }
462
load_stack_from_fp(fp_offset: i64, into_reg: Writable<Reg>, ty: Type) -> Inst463 fn load_stack_from_fp(fp_offset: i64, into_reg: Writable<Reg>, ty: Type) -> Inst {
464 let mem = MemArg::FPOffset(fp_offset);
465 match ty {
466 types::B1
467 | types::B8
468 | types::I8
469 | types::B16
470 | types::I16
471 | types::B32
472 | types::I32
473 | types::B64
474 | types::I64 => Inst::ULoad64 {
475 rd: into_reg,
476 mem,
477 srcloc: None,
478 },
479 types::F32 => Inst::FpuLoad32 {
480 rd: into_reg,
481 mem,
482 srcloc: None,
483 },
484 types::F64 => Inst::FpuLoad64 {
485 rd: into_reg,
486 mem,
487 srcloc: None,
488 },
489 _ => unimplemented!("load_stack_from_fp({})", ty),
490 }
491 }
492
store_stack(mem: MemArg, from_reg: Reg, ty: Type) -> Inst493 fn store_stack(mem: MemArg, from_reg: Reg, ty: Type) -> Inst {
494 debug_assert!(match &mem {
495 MemArg::SPOffset(off) => SImm9::maybe_from_i64(*off).is_some(),
496 _ => true,
497 });
498 match ty {
499 types::B1
500 | types::B8
501 | types::I8
502 | types::B16
503 | types::I16
504 | types::B32
505 | types::I32
506 | types::B64
507 | types::I64 => Inst::Store64 {
508 rd: from_reg,
509 mem,
510 srcloc: None,
511 },
512 types::F32 => Inst::FpuStore32 {
513 rd: from_reg,
514 mem,
515 srcloc: None,
516 },
517 types::F64 => Inst::FpuStore64 {
518 rd: from_reg,
519 mem,
520 srcloc: None,
521 },
522 _ => unimplemented!("store_stack({})", ty),
523 }
524 }
525
store_stack_fp(fp_offset: i64, from_reg: Reg, ty: Type) -> Inst526 fn store_stack_fp(fp_offset: i64, from_reg: Reg, ty: Type) -> Inst {
527 store_stack(MemArg::FPOffset(fp_offset), from_reg, ty)
528 }
529
store_stack_sp<C: LowerCtx<I = Inst>>( ctx: &mut C, sp_offset: i64, from_reg: Reg, ty: Type, ) -> Vec<Inst>530 fn store_stack_sp<C: LowerCtx<I = Inst>>(
531 ctx: &mut C,
532 sp_offset: i64,
533 from_reg: Reg,
534 ty: Type,
535 ) -> Vec<Inst> {
536 if SImm9::maybe_from_i64(sp_offset).is_some() {
537 vec![store_stack(MemArg::SPOffset(sp_offset), from_reg, ty)]
538 } else {
539 // mem_finalize will try to generate an add, but in an addition, x31 is the zero register,
540 // not sp! So we have to synthesize the full add here.
541 let tmp1 = ctx.tmp(RegClass::I64, I64);
542 let tmp2 = ctx.tmp(RegClass::I64, I64);
543 let mut result = Vec::new();
544 // tmp1 := sp
545 result.push(Inst::Mov {
546 rd: tmp1,
547 rm: stack_reg(),
548 });
549 // tmp2 := offset
550 for inst in Inst::load_constant(tmp2, sp_offset as u64) {
551 result.push(inst);
552 }
553 // tmp1 := add tmp1, tmp2
554 result.push(Inst::AluRRR {
555 alu_op: ALUOp::Add64,
556 rd: tmp1,
557 rn: tmp1.to_reg(),
558 rm: tmp2.to_reg(),
559 });
560 // Actual store.
561 result.push(store_stack(
562 MemArg::Unscaled(tmp1.to_reg(), SImm9::maybe_from_i64(0).unwrap()),
563 from_reg,
564 ty,
565 ));
566 result
567 }
568 }
569
is_callee_save(call_conv: isa::CallConv, r: RealReg) -> bool570 fn is_callee_save(call_conv: isa::CallConv, r: RealReg) -> bool {
571 if call_conv.extends_baldrdash() {
572 match r.get_class() {
573 RegClass::I64 => {
574 let enc = r.get_hw_encoding();
575 return BALDRDASH_JIT_CALLEE_SAVED_GPR[enc];
576 }
577 RegClass::V128 => {
578 let enc = r.get_hw_encoding();
579 return BALDRDASH_JIT_CALLEE_SAVED_FPU[enc];
580 }
581 _ => unimplemented!("baldrdash callee saved on non-i64 reg classes"),
582 };
583 }
584
585 match r.get_class() {
586 RegClass::I64 => {
587 // x19 - x28 inclusive are callee-saves.
588 r.get_hw_encoding() >= 19 && r.get_hw_encoding() <= 28
589 }
590 RegClass::V128 => {
591 // v8 - v15 inclusive are callee-saves.
592 r.get_hw_encoding() >= 8 && r.get_hw_encoding() <= 15
593 }
594 _ => panic!("Unexpected RegClass"),
595 }
596 }
597
get_callee_saves( call_conv: isa::CallConv, regs: Vec<Writable<RealReg>>, ) -> (Vec<Writable<RealReg>>, Vec<Writable<RealReg>>)598 fn get_callee_saves(
599 call_conv: isa::CallConv,
600 regs: Vec<Writable<RealReg>>,
601 ) -> (Vec<Writable<RealReg>>, Vec<Writable<RealReg>>) {
602 let mut int_saves = vec![];
603 let mut vec_saves = vec![];
604 for reg in regs.into_iter() {
605 if is_callee_save(call_conv, reg.to_reg()) {
606 match reg.to_reg().get_class() {
607 RegClass::I64 => int_saves.push(reg),
608 RegClass::V128 => vec_saves.push(reg),
609 _ => panic!("Unexpected RegClass"),
610 }
611 }
612 }
613 (int_saves, vec_saves)
614 }
615
is_caller_save(call_conv: isa::CallConv, r: RealReg) -> bool616 fn is_caller_save(call_conv: isa::CallConv, r: RealReg) -> bool {
617 if call_conv.extends_baldrdash() {
618 match r.get_class() {
619 RegClass::I64 => {
620 let enc = r.get_hw_encoding();
621 if !BALDRDASH_JIT_CALLEE_SAVED_GPR[enc] {
622 return true;
623 }
624 // Otherwise, fall through to preserve native's ABI caller-saved.
625 }
626 RegClass::V128 => {
627 let enc = r.get_hw_encoding();
628 if !BALDRDASH_JIT_CALLEE_SAVED_FPU[enc] {
629 return true;
630 }
631 // Otherwise, fall through to preserve native's ABI caller-saved.
632 }
633 _ => unimplemented!("baldrdash callee saved on non-i64 reg classes"),
634 };
635 }
636
637 match r.get_class() {
638 RegClass::I64 => {
639 // x0 - x17 inclusive are caller-saves.
640 r.get_hw_encoding() <= 17
641 }
642 RegClass::V128 => {
643 // v0 - v7 inclusive and v16 - v31 inclusive are caller-saves.
644 r.get_hw_encoding() <= 7 || (r.get_hw_encoding() >= 16 && r.get_hw_encoding() <= 31)
645 }
646 _ => panic!("Unexpected RegClass"),
647 }
648 }
649
get_caller_saves_set(call_conv: isa::CallConv) -> Set<Writable<Reg>>650 fn get_caller_saves_set(call_conv: isa::CallConv) -> Set<Writable<Reg>> {
651 let mut set = Set::empty();
652 for i in 0..29 {
653 let x = writable_xreg(i);
654 if is_caller_save(call_conv, x.to_reg().to_real_reg()) {
655 set.insert(x);
656 }
657 }
658 for i in 0..32 {
659 let v = writable_vreg(i);
660 if is_caller_save(call_conv, v.to_reg().to_real_reg()) {
661 set.insert(v);
662 }
663 }
664 set
665 }
666
667 impl ABIBody for AArch64ABIBody {
668 type I = Inst;
669
flags(&self) -> &settings::Flags670 fn flags(&self) -> &settings::Flags {
671 &self.flags
672 }
673
liveins(&self) -> Set<RealReg>674 fn liveins(&self) -> Set<RealReg> {
675 let mut set: Set<RealReg> = Set::empty();
676 for &arg in &self.sig.args {
677 if let ABIArg::Reg(r, _) = arg {
678 set.insert(r);
679 }
680 }
681 set
682 }
683
liveouts(&self) -> Set<RealReg>684 fn liveouts(&self) -> Set<RealReg> {
685 let mut set: Set<RealReg> = Set::empty();
686 for &ret in &self.sig.rets {
687 if let ABIArg::Reg(r, _) = ret {
688 set.insert(r);
689 }
690 }
691 set
692 }
693
num_args(&self) -> usize694 fn num_args(&self) -> usize {
695 self.sig.args.len()
696 }
697
num_retvals(&self) -> usize698 fn num_retvals(&self) -> usize {
699 self.sig.rets.len()
700 }
701
num_stackslots(&self) -> usize702 fn num_stackslots(&self) -> usize {
703 self.stackslots.len()
704 }
705
gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst706 fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst {
707 match &self.sig.args[idx] {
708 &ABIArg::Reg(r, ty) => Inst::gen_move(into_reg, r.to_reg(), ty),
709 &ABIArg::Stack(off, ty) => load_stack_from_fp(off + self.frame_size(), into_reg, ty),
710 }
711 }
712
gen_copy_reg_to_retval( &self, idx: usize, from_reg: Writable<Reg>, ext: ArgumentExtension, ) -> Vec<Inst>713 fn gen_copy_reg_to_retval(
714 &self,
715 idx: usize,
716 from_reg: Writable<Reg>,
717 ext: ArgumentExtension,
718 ) -> Vec<Inst> {
719 let mut ret = Vec::new();
720 match &self.sig.rets[idx] {
721 &ABIArg::Reg(r, ty) => {
722 let from_bits = aarch64::lower::ty_bits(ty) as u8;
723 let dest_reg = Writable::from_reg(r.to_reg());
724 match (ext, from_bits) {
725 (ArgumentExtension::Uext, n) if n < 64 => {
726 ret.push(Inst::Extend {
727 rd: dest_reg,
728 rn: from_reg.to_reg(),
729 signed: false,
730 from_bits,
731 to_bits: 64,
732 });
733 }
734 (ArgumentExtension::Sext, n) if n < 64 => {
735 ret.push(Inst::Extend {
736 rd: dest_reg,
737 rn: from_reg.to_reg(),
738 signed: true,
739 from_bits,
740 to_bits: 64,
741 });
742 }
743 _ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)),
744 };
745 }
746 &ABIArg::Stack(off, ty) => {
747 let from_bits = aarch64::lower::ty_bits(ty) as u8;
748 // Trash the from_reg; it should be its last use.
749 match (ext, from_bits) {
750 (ArgumentExtension::Uext, n) if n < 64 => {
751 ret.push(Inst::Extend {
752 rd: from_reg,
753 rn: from_reg.to_reg(),
754 signed: false,
755 from_bits,
756 to_bits: 64,
757 });
758 }
759 (ArgumentExtension::Sext, n) if n < 64 => {
760 ret.push(Inst::Extend {
761 rd: from_reg,
762 rn: from_reg.to_reg(),
763 signed: true,
764 from_bits,
765 to_bits: 64,
766 });
767 }
768 _ => {}
769 };
770 ret.push(store_stack_fp(
771 off + self.frame_size(),
772 from_reg.to_reg(),
773 ty,
774 ))
775 }
776 }
777 ret
778 }
779
gen_ret(&self) -> Inst780 fn gen_ret(&self) -> Inst {
781 Inst::Ret {}
782 }
783
gen_epilogue_placeholder(&self) -> Inst784 fn gen_epilogue_placeholder(&self) -> Inst {
785 Inst::EpiloguePlaceholder {}
786 }
787
set_num_spillslots(&mut self, slots: usize)788 fn set_num_spillslots(&mut self, slots: usize) {
789 self.spillslots = Some(slots);
790 }
791
set_clobbered(&mut self, clobbered: Set<Writable<RealReg>>)792 fn set_clobbered(&mut self, clobbered: Set<Writable<RealReg>>) {
793 self.clobbered = clobbered;
794 }
795
load_stackslot( &self, slot: StackSlot, offset: u32, ty: Type, into_reg: Writable<Reg>, ) -> Inst796 fn load_stackslot(
797 &self,
798 slot: StackSlot,
799 offset: u32,
800 ty: Type,
801 into_reg: Writable<Reg>,
802 ) -> Inst {
803 // Offset from beginning of stackslot area, which is at FP - stackslots_size.
804 let stack_off = self.stackslots[slot.as_u32() as usize] as i64;
805 let fp_off: i64 = -(self.stackslots_size as i64) + stack_off + (offset as i64);
806 load_stack_from_fp(fp_off, into_reg, ty)
807 }
808
store_stackslot(&self, slot: StackSlot, offset: u32, ty: Type, from_reg: Reg) -> Inst809 fn store_stackslot(&self, slot: StackSlot, offset: u32, ty: Type, from_reg: Reg) -> Inst {
810 // Offset from beginning of stackslot area, which is at FP - stackslots_size.
811 let stack_off = self.stackslots[slot.as_u32() as usize] as i64;
812 let fp_off: i64 = -(self.stackslots_size as i64) + stack_off + (offset as i64);
813 store_stack_fp(fp_off, from_reg, ty)
814 }
815
stackslot_addr(&self, slot: StackSlot, offset: u32, into_reg: Writable<Reg>) -> Inst816 fn stackslot_addr(&self, slot: StackSlot, offset: u32, into_reg: Writable<Reg>) -> Inst {
817 // Offset from beginning of stackslot area, which is at FP - stackslots_size.
818 let stack_off = self.stackslots[slot.as_u32() as usize] as i64;
819 let fp_off: i64 = -(self.stackslots_size as i64) + stack_off + (offset as i64);
820 Inst::LoadAddr {
821 rd: into_reg,
822 mem: MemArg::FPOffset(fp_off),
823 }
824 }
825
826 // Load from a spillslot.
load_spillslot(&self, slot: SpillSlot, ty: Type, into_reg: Writable<Reg>) -> Inst827 fn load_spillslot(&self, slot: SpillSlot, ty: Type, into_reg: Writable<Reg>) -> Inst {
828 // Note that when spills/fills are generated, we don't yet know how many
829 // spillslots there will be, so we allocate *downward* from the beginning
830 // of the stackslot area. Hence: FP - stackslot_size - 8*spillslot -
831 // sizeof(ty).
832 let islot = slot.get() as i64;
833 let ty_size = self.get_spillslot_size(into_reg.to_reg().get_class(), ty) * 8;
834 let fp_off: i64 = -(self.stackslots_size as i64) - (8 * islot) - ty_size as i64;
835 load_stack_from_fp(fp_off, into_reg, ty)
836 }
837
838 // Store to a spillslot.
store_spillslot(&self, slot: SpillSlot, ty: Type, from_reg: Reg) -> Inst839 fn store_spillslot(&self, slot: SpillSlot, ty: Type, from_reg: Reg) -> Inst {
840 let islot = slot.get() as i64;
841 let ty_size = self.get_spillslot_size(from_reg.get_class(), ty) * 8;
842 let fp_off: i64 = -(self.stackslots_size as i64) - (8 * islot) - ty_size as i64;
843 store_stack_fp(fp_off, from_reg, ty)
844 }
845
gen_prologue(&mut self) -> Vec<Inst>846 fn gen_prologue(&mut self) -> Vec<Inst> {
847 let mut insts = vec![];
848 if !self.call_conv.extends_baldrdash() {
849 // stp fp (x29), lr (x30), [sp, #-16]!
850 insts.push(Inst::StoreP64 {
851 rt: fp_reg(),
852 rt2: link_reg(),
853 mem: PairMemArg::PreIndexed(
854 writable_stack_reg(),
855 SImm7Scaled::maybe_from_i64(-16, types::I64).unwrap(),
856 ),
857 });
858 // mov fp (x29), sp. This uses the ADDI rd, rs, 0 form of `MOV` because
859 // the usual encoding (`ORR`) does not work with SP.
860 insts.push(Inst::AluRRImm12 {
861 alu_op: ALUOp::Add64,
862 rd: writable_fp_reg(),
863 rn: stack_reg(),
864 imm12: Imm12 {
865 bits: 0,
866 shift12: false,
867 },
868 });
869 }
870
871 let mut total_stacksize = self.stackslots_size + 8 * self.spillslots.unwrap() as u32;
872 if self.call_conv.extends_baldrdash() {
873 debug_assert!(
874 !self.flags.enable_probestack(),
875 "baldrdash does not expect cranelift to emit stack probes"
876 );
877 total_stacksize += self.flags.baldrdash_prologue_words() as u32 * 8;
878 }
879 let total_stacksize = (total_stacksize + 15) & !15; // 16-align the stack.
880
881 if !self.call_conv.extends_baldrdash() {
882 // Leaf functions with zero stack don't need a stack check if one's
883 // specified, otherwise always insert the stack check.
884 if total_stacksize > 0 || !self.is_leaf {
885 if let Some((reg, stack_limit_load)) = &self.stack_limit {
886 insts.extend_from_slice(stack_limit_load);
887 self.insert_stack_check(*reg, total_stacksize, &mut insts);
888 }
889 }
890 if total_stacksize > 0 {
891 // sub sp, sp, #total_stacksize
892 if let Some(imm12) = Imm12::maybe_from_u64(total_stacksize as u64) {
893 let sub_inst = Inst::AluRRImm12 {
894 alu_op: ALUOp::Sub64,
895 rd: writable_stack_reg(),
896 rn: stack_reg(),
897 imm12,
898 };
899 insts.push(sub_inst);
900 } else {
901 let tmp = writable_spilltmp_reg();
902 let const_inst = Inst::LoadConst64 {
903 rd: tmp,
904 const_data: total_stacksize as u64,
905 };
906 let sub_inst = Inst::AluRRRExtend {
907 alu_op: ALUOp::Sub64,
908 rd: writable_stack_reg(),
909 rn: stack_reg(),
910 rm: tmp.to_reg(),
911 extendop: ExtendOp::UXTX,
912 };
913 insts.push(const_inst);
914 insts.push(sub_inst);
915 }
916 }
917 }
918
919 // Save clobbered registers.
920 let (clobbered_int, clobbered_vec) =
921 get_callee_saves(self.call_conv, self.clobbered.to_vec());
922 for reg_pair in clobbered_int.chunks(2) {
923 let (r1, r2) = if reg_pair.len() == 2 {
924 // .to_reg().to_reg(): Writable<RealReg> --> RealReg --> Reg
925 (reg_pair[0].to_reg().to_reg(), reg_pair[1].to_reg().to_reg())
926 } else {
927 (reg_pair[0].to_reg().to_reg(), zero_reg())
928 };
929
930 debug_assert!(r1.get_class() == RegClass::I64);
931 debug_assert!(r2.get_class() == RegClass::I64);
932
933 // stp r1, r2, [sp, #-16]!
934 insts.push(Inst::StoreP64 {
935 rt: r1,
936 rt2: r2,
937 mem: PairMemArg::PreIndexed(
938 writable_stack_reg(),
939 SImm7Scaled::maybe_from_i64(-16, types::I64).unwrap(),
940 ),
941 });
942 }
943 let vec_save_bytes = clobbered_vec.len() * 16;
944 if vec_save_bytes != 0 {
945 insts.push(Inst::AluRRImm12 {
946 alu_op: ALUOp::Sub64,
947 rd: writable_stack_reg(),
948 rn: stack_reg(),
949 imm12: Imm12::maybe_from_u64(vec_save_bytes as u64).unwrap(),
950 });
951 }
952 for (i, reg) in clobbered_vec.iter().enumerate() {
953 insts.push(Inst::FpuStore128 {
954 rd: reg.to_reg().to_reg(),
955 mem: MemArg::Unscaled(stack_reg(), SImm9::maybe_from_i64((i * 16) as i64).unwrap()),
956 srcloc: None,
957 });
958 }
959
960 self.frame_size = Some(total_stacksize);
961 insts
962 }
963
gen_epilogue(&self) -> Vec<Inst>964 fn gen_epilogue(&self) -> Vec<Inst> {
965 let mut insts = vec![];
966
967 // Restore clobbered registers.
968 let (clobbered_int, clobbered_vec) =
969 get_callee_saves(self.call_conv, self.clobbered.to_vec());
970
971 for (i, reg) in clobbered_vec.iter().enumerate() {
972 insts.push(Inst::FpuLoad128 {
973 rd: Writable::from_reg(reg.to_reg().to_reg()),
974 mem: MemArg::Unscaled(stack_reg(), SImm9::maybe_from_i64((i * 16) as i64).unwrap()),
975 srcloc: None,
976 });
977 }
978 let vec_save_bytes = clobbered_vec.len() * 16;
979 if vec_save_bytes != 0 {
980 insts.push(Inst::AluRRImm12 {
981 alu_op: ALUOp::Add64,
982 rd: writable_stack_reg(),
983 rn: stack_reg(),
984 imm12: Imm12::maybe_from_u64(vec_save_bytes as u64).unwrap(),
985 });
986 }
987
988 for reg_pair in clobbered_int.chunks(2).rev() {
989 let (r1, r2) = if reg_pair.len() == 2 {
990 (
991 reg_pair[0].map(|r| r.to_reg()),
992 reg_pair[1].map(|r| r.to_reg()),
993 )
994 } else {
995 (reg_pair[0].map(|r| r.to_reg()), writable_zero_reg())
996 };
997
998 debug_assert!(r1.to_reg().get_class() == RegClass::I64);
999 debug_assert!(r2.to_reg().get_class() == RegClass::I64);
1000
1001 // ldp r1, r2, [sp], #16
1002 insts.push(Inst::LoadP64 {
1003 rt: r1,
1004 rt2: r2,
1005 mem: PairMemArg::PostIndexed(
1006 writable_stack_reg(),
1007 SImm7Scaled::maybe_from_i64(16, types::I64).unwrap(),
1008 ),
1009 });
1010 }
1011
1012 if !self.call_conv.extends_baldrdash() {
1013 // The MOV (alias of ORR) interprets x31 as XZR, so use an ADD here.
1014 // MOV to SP is an alias of ADD.
1015 insts.push(Inst::AluRRImm12 {
1016 alu_op: ALUOp::Add64,
1017 rd: writable_stack_reg(),
1018 rn: fp_reg(),
1019 imm12: Imm12 {
1020 bits: 0,
1021 shift12: false,
1022 },
1023 });
1024 insts.push(Inst::LoadP64 {
1025 rt: writable_fp_reg(),
1026 rt2: writable_link_reg(),
1027 mem: PairMemArg::PostIndexed(
1028 writable_stack_reg(),
1029 SImm7Scaled::maybe_from_i64(16, types::I64).unwrap(),
1030 ),
1031 });
1032 insts.push(Inst::Ret {});
1033 }
1034
1035 debug!("Epilogue: {:?}", insts);
1036 insts
1037 }
1038
frame_size(&self) -> u321039 fn frame_size(&self) -> u32 {
1040 self.frame_size
1041 .expect("frame size not computed before prologue generation")
1042 }
1043
get_spillslot_size(&self, rc: RegClass, ty: Type) -> u321044 fn get_spillslot_size(&self, rc: RegClass, ty: Type) -> u32 {
1045 // We allocate in terms of 8-byte slots.
1046 match (rc, ty) {
1047 (RegClass::I64, _) => 1,
1048 (RegClass::V128, F32) | (RegClass::V128, F64) => 1,
1049 (RegClass::V128, _) => 2,
1050 _ => panic!("Unexpected register class!"),
1051 }
1052 }
1053
gen_spill(&self, to_slot: SpillSlot, from_reg: RealReg, ty: Type) -> Inst1054 fn gen_spill(&self, to_slot: SpillSlot, from_reg: RealReg, ty: Type) -> Inst {
1055 self.store_spillslot(to_slot, ty, from_reg.to_reg())
1056 }
1057
gen_reload(&self, to_reg: Writable<RealReg>, from_slot: SpillSlot, ty: Type) -> Inst1058 fn gen_reload(&self, to_reg: Writable<RealReg>, from_slot: SpillSlot, ty: Type) -> Inst {
1059 self.load_spillslot(from_slot, ty, to_reg.map(|r| r.to_reg()))
1060 }
1061 }
1062
1063 enum CallDest {
1064 ExtName(ir::ExternalName),
1065 Reg(Reg),
1066 }
1067
1068 /// AArch64 ABI object for a function call.
1069 pub struct AArch64ABICall {
1070 sig: ABISig,
1071 uses: Set<Reg>,
1072 defs: Set<Writable<Reg>>,
1073 dest: CallDest,
1074 loc: ir::SourceLoc,
1075 opcode: ir::Opcode,
1076 }
1077
abisig_to_uses_and_defs(sig: &ABISig) -> (Set<Reg>, Set<Writable<Reg>>)1078 fn abisig_to_uses_and_defs(sig: &ABISig) -> (Set<Reg>, Set<Writable<Reg>>) {
1079 // Compute uses: all arg regs.
1080 let mut uses = Set::empty();
1081 for arg in &sig.args {
1082 match arg {
1083 &ABIArg::Reg(reg, _) => uses.insert(reg.to_reg()),
1084 _ => {}
1085 }
1086 }
1087
1088 // Compute defs: all retval regs, and all caller-save (clobbered) regs.
1089 let mut defs = get_caller_saves_set(sig.call_conv);
1090 for ret in &sig.rets {
1091 match ret {
1092 &ABIArg::Reg(reg, _) => defs.insert(Writable::from_reg(reg.to_reg())),
1093 _ => {}
1094 }
1095 }
1096
1097 (uses, defs)
1098 }
1099
1100 impl AArch64ABICall {
1101 /// Create a callsite ABI object for a call directly to the specified function.
from_func( sig: &ir::Signature, extname: &ir::ExternalName, loc: ir::SourceLoc, ) -> AArch64ABICall1102 pub fn from_func(
1103 sig: &ir::Signature,
1104 extname: &ir::ExternalName,
1105 loc: ir::SourceLoc,
1106 ) -> AArch64ABICall {
1107 let sig = ABISig::from_func_sig(sig);
1108 let (uses, defs) = abisig_to_uses_and_defs(&sig);
1109 AArch64ABICall {
1110 sig,
1111 uses,
1112 defs,
1113 dest: CallDest::ExtName(extname.clone()),
1114 loc,
1115 opcode: ir::Opcode::Call,
1116 }
1117 }
1118
1119 /// Create a callsite ABI object for a call to a function pointer with the
1120 /// given signature.
from_ptr( sig: &ir::Signature, ptr: Reg, loc: ir::SourceLoc, opcode: ir::Opcode, ) -> AArch64ABICall1121 pub fn from_ptr(
1122 sig: &ir::Signature,
1123 ptr: Reg,
1124 loc: ir::SourceLoc,
1125 opcode: ir::Opcode,
1126 ) -> AArch64ABICall {
1127 let sig = ABISig::from_func_sig(sig);
1128 let (uses, defs) = abisig_to_uses_and_defs(&sig);
1129 AArch64ABICall {
1130 sig,
1131 uses,
1132 defs,
1133 dest: CallDest::Reg(ptr),
1134 loc,
1135 opcode,
1136 }
1137 }
1138 }
1139
adjust_stack(amt: u64, is_sub: bool) -> Vec<Inst>1140 fn adjust_stack(amt: u64, is_sub: bool) -> Vec<Inst> {
1141 if amt > 0 {
1142 let alu_op = if is_sub { ALUOp::Sub64 } else { ALUOp::Add64 };
1143 if let Some(imm12) = Imm12::maybe_from_u64(amt) {
1144 vec![Inst::AluRRImm12 {
1145 alu_op,
1146 rd: writable_stack_reg(),
1147 rn: stack_reg(),
1148 imm12,
1149 }]
1150 } else {
1151 let const_load = Inst::LoadConst64 {
1152 rd: writable_spilltmp_reg(),
1153 const_data: amt,
1154 };
1155 let adj = Inst::AluRRRExtend {
1156 alu_op,
1157 rd: writable_stack_reg(),
1158 rn: stack_reg(),
1159 rm: spilltmp_reg(),
1160 extendop: ExtendOp::UXTX,
1161 };
1162 vec![const_load, adj]
1163 }
1164 } else {
1165 vec![]
1166 }
1167 }
1168
1169 impl ABICall for AArch64ABICall {
1170 type I = Inst;
1171
num_args(&self) -> usize1172 fn num_args(&self) -> usize {
1173 self.sig.args.len()
1174 }
1175
gen_stack_pre_adjust(&self) -> Vec<Inst>1176 fn gen_stack_pre_adjust(&self) -> Vec<Inst> {
1177 adjust_stack(self.sig.stack_arg_space as u64, /* is_sub = */ true)
1178 }
1179
gen_stack_post_adjust(&self) -> Vec<Inst>1180 fn gen_stack_post_adjust(&self) -> Vec<Inst> {
1181 adjust_stack(self.sig.stack_arg_space as u64, /* is_sub = */ false)
1182 }
1183
gen_copy_reg_to_arg<C: LowerCtx<I = Self::I>>( &self, ctx: &mut C, idx: usize, from_reg: Reg, ) -> Vec<Inst>1184 fn gen_copy_reg_to_arg<C: LowerCtx<I = Self::I>>(
1185 &self,
1186 ctx: &mut C,
1187 idx: usize,
1188 from_reg: Reg,
1189 ) -> Vec<Inst> {
1190 match &self.sig.args[idx] {
1191 &ABIArg::Reg(reg, ty) => vec![Inst::gen_move(
1192 Writable::from_reg(reg.to_reg()),
1193 from_reg,
1194 ty,
1195 )],
1196 &ABIArg::Stack(off, ty) => store_stack_sp(ctx, off, from_reg, ty),
1197 }
1198 }
1199
gen_copy_retval_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst1200 fn gen_copy_retval_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst {
1201 match &self.sig.rets[idx] {
1202 &ABIArg::Reg(reg, ty) => Inst::gen_move(into_reg, reg.to_reg(), ty),
1203 _ => unimplemented!(),
1204 }
1205 }
1206
gen_call(&self) -> Vec<Inst>1207 fn gen_call(&self) -> Vec<Inst> {
1208 let (uses, defs) = (self.uses.clone(), self.defs.clone());
1209 match &self.dest {
1210 &CallDest::ExtName(ref name) => vec![Inst::Call {
1211 dest: name.clone(),
1212 uses,
1213 defs,
1214 loc: self.loc,
1215 opcode: self.opcode,
1216 }],
1217 &CallDest::Reg(reg) => vec![Inst::CallInd {
1218 rn: reg,
1219 uses,
1220 defs,
1221 loc: self.loc,
1222 opcode: self.opcode,
1223 }],
1224 }
1225 }
1226 }
1227