1 //! This module defines aarch64-specific machine instruction types.
2
3 // Some variants are not constructed, but we still want them as options in the future.
4 #![allow(dead_code)]
5
6 use crate::binemit::CodeOffset;
7 use crate::ir::types::{
8 B1, B16, B32, B64, B8, B8X16, F32, F32X2, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS,
9 };
10 use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
11 use crate::machinst::*;
12 use crate::{settings, CodegenError, CodegenResult};
13
14 use regalloc::{RealRegUniverse, Reg, RegClass, SpillSlot, VirtualReg, Writable};
15 use regalloc::{RegUsageCollector, RegUsageMapper};
16
17 use alloc::boxed::Box;
18 use alloc::vec::Vec;
19 use smallvec::{smallvec, SmallVec};
20 use std::string::{String, ToString};
21
22 pub mod regs;
23 pub use self::regs::*;
24 pub mod imms;
25 pub use self::imms::*;
26 pub mod args;
27 pub use self::args::*;
28 pub mod emit;
29 pub use self::emit::*;
30
31 #[cfg(test)]
32 mod emit_tests;
33
34 //=============================================================================
35 // Instructions (top level): definition
36
37 /// An ALU operation. This can be paired with several instruction formats
38 /// below (see `Inst`) in any combination.
39 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
40 pub enum ALUOp {
41 Add32,
42 Add64,
43 Sub32,
44 Sub64,
45 Orr32,
46 Orr64,
47 /// NOR
48 OrrNot32,
49 /// NOR
50 OrrNot64,
51 And32,
52 And64,
53 /// NAND
54 AndNot32,
55 /// NAND
56 AndNot64,
57 /// XOR (AArch64 calls this "EOR")
58 Eor32,
59 /// XOR (AArch64 calls this "EOR")
60 Eor64,
61 /// XNOR (AArch64 calls this "EOR-NOT")
62 EorNot32,
63 /// XNOR (AArch64 calls this "EOR-NOT")
64 EorNot64,
65 /// Add, setting flags
66 AddS32,
67 /// Add, setting flags
68 AddS64,
69 /// Sub, setting flags
70 SubS32,
71 /// Sub, setting flags
72 SubS64,
73 /// Sub, setting flags, using extended registers
74 SubS64XR,
75 /// Multiply-add
76 MAdd32,
77 /// Multiply-add
78 MAdd64,
79 /// Multiply-sub
80 MSub32,
81 /// Multiply-sub
82 MSub64,
83 /// Signed multiply, high-word result
84 SMulH,
85 /// Unsigned multiply, high-word result
86 UMulH,
87 SDiv64,
88 UDiv64,
89 RotR32,
90 RotR64,
91 Lsr32,
92 Lsr64,
93 Asr32,
94 Asr64,
95 Lsl32,
96 Lsl64,
97 }
98
99 /// A floating-point unit (FPU) operation with one arg.
100 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
101 pub enum FPUOp1 {
102 Abs32,
103 Abs64,
104 Neg32,
105 Neg64,
106 Sqrt32,
107 Sqrt64,
108 Cvt32To64,
109 Cvt64To32,
110 }
111
112 /// A floating-point unit (FPU) operation with two args.
113 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
114 pub enum FPUOp2 {
115 Add32,
116 Add64,
117 Sub32,
118 Sub64,
119 Mul32,
120 Mul64,
121 Div32,
122 Div64,
123 Max32,
124 Max64,
125 Min32,
126 Min64,
127 }
128
129 /// A floating-point unit (FPU) operation with two args, a register and an immediate.
130 #[derive(Copy, Clone, Debug)]
131 pub enum FPUOpRI {
132 /// Unsigned right shift. Rd = Rn << #imm
133 UShr32(FPURightShiftImm),
134 /// Unsigned right shift. Rd = Rn << #imm
135 UShr64(FPURightShiftImm),
136 /// Shift left and insert. Rd |= Rn << #imm
137 Sli32(FPULeftShiftImm),
138 /// Shift left and insert. Rd |= Rn << #imm
139 Sli64(FPULeftShiftImm),
140 }
141
142 /// A floating-point unit (FPU) operation with three args.
143 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
144 pub enum FPUOp3 {
145 MAdd32,
146 MAdd64,
147 }
148
149 /// A conversion from an FP to an integer value.
150 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
151 pub enum FpuToIntOp {
152 F32ToU32,
153 F32ToI32,
154 F32ToU64,
155 F32ToI64,
156 F64ToU32,
157 F64ToI32,
158 F64ToU64,
159 F64ToI64,
160 }
161
162 /// A conversion from an integer to an FP value.
163 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
164 pub enum IntToFpuOp {
165 U32ToF32,
166 I32ToF32,
167 U32ToF64,
168 I32ToF64,
169 U64ToF32,
170 I64ToF32,
171 U64ToF64,
172 I64ToF64,
173 }
174
175 /// Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to
176 /// nearest, and for 32- or 64-bit FP values.
177 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
178 pub enum FpuRoundMode {
179 Minus32,
180 Minus64,
181 Plus32,
182 Plus64,
183 Zero32,
184 Zero64,
185 Nearest32,
186 Nearest64,
187 }
188
189 /// A vector ALU operation.
190 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
191 pub enum VecALUOp {
192 /// Signed saturating add
193 SQAddScalar,
194 /// Unsigned saturating add
195 UQAddScalar,
196 /// Signed saturating subtract
197 SQSubScalar,
198 /// Unsigned saturating subtract
199 UQSubScalar,
200 /// Compare bitwise equal
201 Cmeq,
202 /// Compare signed greater than or equal
203 Cmge,
204 /// Compare signed greater than
205 Cmgt,
206 /// Compare unsigned higher
207 Cmhs,
208 /// Compare unsigned higher or same
209 Cmhi,
210 }
211
212 /// A Vector miscellaneous operation with two registers.
213 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
214 pub enum VecMisc2 {
215 /// Bitwise NOT.
216 Not,
217 }
218
219 /// An operation on the bits of a register. This can be paired with several instruction formats
220 /// below (see `Inst`) in any combination.
221 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
222 pub enum BitOp {
223 /// Bit reverse
224 RBit32,
225 /// Bit reverse
226 RBit64,
227 Clz32,
228 Clz64,
229 Cls32,
230 Cls64,
231 }
232
233 impl BitOp {
234 /// What is the opcode's native width?
inst_size(&self) -> InstSize235 pub fn inst_size(&self) -> InstSize {
236 match self {
237 BitOp::RBit32 | BitOp::Clz32 | BitOp::Cls32 => InstSize::Size32,
238 _ => InstSize::Size64,
239 }
240 }
241
242 /// Get the assembly mnemonic for this opcode.
op_str(&self) -> &'static str243 pub fn op_str(&self) -> &'static str {
244 match self {
245 BitOp::RBit32 | BitOp::RBit64 => "rbit",
246 BitOp::Clz32 | BitOp::Clz64 => "clz",
247 BitOp::Cls32 | BitOp::Cls64 => "cls",
248 }
249 }
250 }
251
252 impl From<(Opcode, Type)> for BitOp {
253 /// Get the BitOp from the IR opcode.
from(op_ty: (Opcode, Type)) -> BitOp254 fn from(op_ty: (Opcode, Type)) -> BitOp {
255 match op_ty {
256 (Opcode::Bitrev, I32) => BitOp::RBit32,
257 (Opcode::Bitrev, I64) => BitOp::RBit64,
258 (Opcode::Clz, I32) => BitOp::Clz32,
259 (Opcode::Clz, I64) => BitOp::Clz64,
260 (Opcode::Cls, I32) => BitOp::Cls32,
261 (Opcode::Cls, I64) => BitOp::Cls64,
262 _ => unreachable!("Called with non-bit op!: {:?}", op_ty),
263 }
264 }
265 }
266
267 /// Additional information for (direct) Call instructions, left out of line to lower the size of
268 /// the Inst enum.
269 #[derive(Clone, Debug)]
270 pub struct CallInfo {
271 pub dest: ExternalName,
272 pub uses: Vec<Reg>,
273 pub defs: Vec<Writable<Reg>>,
274 pub loc: SourceLoc,
275 pub opcode: Opcode,
276 }
277
278 /// Additional information for CallInd instructions, left out of line to lower the size of the Inst
279 /// enum.
280 #[derive(Clone, Debug)]
281 pub struct CallIndInfo {
282 pub rn: Reg,
283 pub uses: Vec<Reg>,
284 pub defs: Vec<Writable<Reg>>,
285 pub loc: SourceLoc,
286 pub opcode: Opcode,
287 }
288
289 /// Additional information for JTSequence instructions, left out of line to lower the size of the Inst
290 /// enum.
291 #[derive(Clone, Debug)]
292 pub struct JTSequenceInfo {
293 pub targets: Vec<BranchTarget>,
294 pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
295 }
296
297 /// Instruction formats.
298 #[derive(Clone, Debug)]
299 pub enum Inst {
300 /// A no-op of zero size.
301 Nop0,
302
303 /// A no-op that is one instruction large.
304 Nop4,
305
306 /// An ALU operation with two register sources and a register destination.
307 AluRRR {
308 alu_op: ALUOp,
309 rd: Writable<Reg>,
310 rn: Reg,
311 rm: Reg,
312 },
313 /// An ALU operation with three register sources and a register destination.
314 AluRRRR {
315 alu_op: ALUOp,
316 rd: Writable<Reg>,
317 rn: Reg,
318 rm: Reg,
319 ra: Reg,
320 },
321 /// An ALU operation with a register source and an immediate-12 source, and a register
322 /// destination.
323 AluRRImm12 {
324 alu_op: ALUOp,
325 rd: Writable<Reg>,
326 rn: Reg,
327 imm12: Imm12,
328 },
329 /// An ALU operation with a register source and an immediate-logic source, and a register destination.
330 AluRRImmLogic {
331 alu_op: ALUOp,
332 rd: Writable<Reg>,
333 rn: Reg,
334 imml: ImmLogic,
335 },
336 /// An ALU operation with a register source and an immediate-shiftamt source, and a register destination.
337 AluRRImmShift {
338 alu_op: ALUOp,
339 rd: Writable<Reg>,
340 rn: Reg,
341 immshift: ImmShift,
342 },
343 /// An ALU operation with two register sources, one of which can be shifted, and a register
344 /// destination.
345 AluRRRShift {
346 alu_op: ALUOp,
347 rd: Writable<Reg>,
348 rn: Reg,
349 rm: Reg,
350 shiftop: ShiftOpAndAmt,
351 },
352 /// An ALU operation with two register sources, one of which can be {zero,sign}-extended and
353 /// shifted, and a register destination.
354 AluRRRExtend {
355 alu_op: ALUOp,
356 rd: Writable<Reg>,
357 rn: Reg,
358 rm: Reg,
359 extendop: ExtendOp,
360 },
361
362 /// A bit op instruction with a single register source.
363 BitRR {
364 op: BitOp,
365 rd: Writable<Reg>,
366 rn: Reg,
367 },
368
369 /// An unsigned (zero-extending) 8-bit load.
370 ULoad8 {
371 rd: Writable<Reg>,
372 mem: MemArg,
373 srcloc: Option<SourceLoc>,
374 },
375 /// A signed (sign-extending) 8-bit load.
376 SLoad8 {
377 rd: Writable<Reg>,
378 mem: MemArg,
379 srcloc: Option<SourceLoc>,
380 },
381 /// An unsigned (zero-extending) 16-bit load.
382 ULoad16 {
383 rd: Writable<Reg>,
384 mem: MemArg,
385 srcloc: Option<SourceLoc>,
386 },
387 /// A signed (sign-extending) 16-bit load.
388 SLoad16 {
389 rd: Writable<Reg>,
390 mem: MemArg,
391 srcloc: Option<SourceLoc>,
392 },
393 /// An unsigned (zero-extending) 32-bit load.
394 ULoad32 {
395 rd: Writable<Reg>,
396 mem: MemArg,
397 srcloc: Option<SourceLoc>,
398 },
399 /// A signed (sign-extending) 32-bit load.
400 SLoad32 {
401 rd: Writable<Reg>,
402 mem: MemArg,
403 srcloc: Option<SourceLoc>,
404 },
405 /// A 64-bit load.
406 ULoad64 {
407 rd: Writable<Reg>,
408 mem: MemArg,
409 srcloc: Option<SourceLoc>,
410 },
411
412 /// An 8-bit store.
413 Store8 {
414 rd: Reg,
415 mem: MemArg,
416 srcloc: Option<SourceLoc>,
417 },
418 /// A 16-bit store.
419 Store16 {
420 rd: Reg,
421 mem: MemArg,
422 srcloc: Option<SourceLoc>,
423 },
424 /// A 32-bit store.
425 Store32 {
426 rd: Reg,
427 mem: MemArg,
428 srcloc: Option<SourceLoc>,
429 },
430 /// A 64-bit store.
431 Store64 {
432 rd: Reg,
433 mem: MemArg,
434 srcloc: Option<SourceLoc>,
435 },
436
437 /// A store of a pair of registers.
438 StoreP64 {
439 rt: Reg,
440 rt2: Reg,
441 mem: PairMemArg,
442 },
443 /// A load of a pair of registers.
444 LoadP64 {
445 rt: Writable<Reg>,
446 rt2: Writable<Reg>,
447 mem: PairMemArg,
448 },
449
450 /// A MOV instruction. These are encoded as ORR's (AluRRR form) but we
451 /// keep them separate at the `Inst` level for better pretty-printing
452 /// and faster `is_move()` logic.
453 Mov {
454 rd: Writable<Reg>,
455 rm: Reg,
456 },
457
458 /// A 32-bit MOV. Zeroes the top 32 bits of the destination. This is
459 /// effectively an alias for an unsigned 32-to-64-bit extension.
460 Mov32 {
461 rd: Writable<Reg>,
462 rm: Reg,
463 },
464
465 /// A MOVZ with a 16-bit immediate.
466 MovZ {
467 rd: Writable<Reg>,
468 imm: MoveWideConst,
469 },
470
471 /// A MOVN with a 16-bit immediate.
472 MovN {
473 rd: Writable<Reg>,
474 imm: MoveWideConst,
475 },
476
477 /// A MOVK with a 16-bit immediate.
478 MovK {
479 rd: Writable<Reg>,
480 imm: MoveWideConst,
481 },
482
483 /// A sign- or zero-extend operation.
484 Extend {
485 rd: Writable<Reg>,
486 rn: Reg,
487 signed: bool,
488 from_bits: u8,
489 to_bits: u8,
490 },
491
492 /// A conditional-select operation.
493 CSel {
494 rd: Writable<Reg>,
495 cond: Cond,
496 rn: Reg,
497 rm: Reg,
498 },
499
500 /// A conditional-set operation.
501 CSet {
502 rd: Writable<Reg>,
503 cond: Cond,
504 },
505
506 /// A conditional comparison with an immediate.
507 CCmpImm {
508 size: InstSize,
509 rn: Reg,
510 imm: UImm5,
511 nzcv: NZCV,
512 cond: Cond,
513 },
514
515 /// FPU move. Note that this is distinct from a vector-register
516 /// move; moving just 64 bits seems to be significantly faster.
517 FpuMove64 {
518 rd: Writable<Reg>,
519 rn: Reg,
520 },
521
522 /// Vector register move.
523 FpuMove128 {
524 rd: Writable<Reg>,
525 rn: Reg,
526 },
527
528 /// 1-op FPU instruction.
529 FpuRR {
530 fpu_op: FPUOp1,
531 rd: Writable<Reg>,
532 rn: Reg,
533 },
534
535 /// 2-op FPU instruction.
536 FpuRRR {
537 fpu_op: FPUOp2,
538 rd: Writable<Reg>,
539 rn: Reg,
540 rm: Reg,
541 },
542
543 FpuRRI {
544 fpu_op: FPUOpRI,
545 rd: Writable<Reg>,
546 rn: Reg,
547 },
548
549 /// 3-op FPU instruction.
550 FpuRRRR {
551 fpu_op: FPUOp3,
552 rd: Writable<Reg>,
553 rn: Reg,
554 rm: Reg,
555 ra: Reg,
556 },
557
558 /// FPU comparison, single-precision (32 bit).
559 FpuCmp32 {
560 rn: Reg,
561 rm: Reg,
562 },
563
564 /// FPU comparison, double-precision (64 bit).
565 FpuCmp64 {
566 rn: Reg,
567 rm: Reg,
568 },
569
570 /// Floating-point load, single-precision (32 bit).
571 FpuLoad32 {
572 rd: Writable<Reg>,
573 mem: MemArg,
574 srcloc: Option<SourceLoc>,
575 },
576 /// Floating-point store, single-precision (32 bit).
577 FpuStore32 {
578 rd: Reg,
579 mem: MemArg,
580 srcloc: Option<SourceLoc>,
581 },
582 /// Floating-point load, double-precision (64 bit).
583 FpuLoad64 {
584 rd: Writable<Reg>,
585 mem: MemArg,
586 srcloc: Option<SourceLoc>,
587 },
588 /// Floating-point store, double-precision (64 bit).
589 FpuStore64 {
590 rd: Reg,
591 mem: MemArg,
592 srcloc: Option<SourceLoc>,
593 },
594 /// Floating-point/vector load, 128 bit.
595 FpuLoad128 {
596 rd: Writable<Reg>,
597 mem: MemArg,
598 srcloc: Option<SourceLoc>,
599 },
600 /// Floating-point/vector store, 128 bit.
601 FpuStore128 {
602 rd: Reg,
603 mem: MemArg,
604 srcloc: Option<SourceLoc>,
605 },
606
607 LoadFpuConst32 {
608 rd: Writable<Reg>,
609 const_data: f32,
610 },
611
612 LoadFpuConst64 {
613 rd: Writable<Reg>,
614 const_data: f64,
615 },
616
617 LoadFpuConst128 {
618 rd: Writable<Reg>,
619 const_data: u128,
620 },
621
622 /// Conversion: FP -> integer.
623 FpuToInt {
624 op: FpuToIntOp,
625 rd: Writable<Reg>,
626 rn: Reg,
627 },
628
629 /// Conversion: integer -> FP.
630 IntToFpu {
631 op: IntToFpuOp,
632 rd: Writable<Reg>,
633 rn: Reg,
634 },
635
636 /// FP conditional select, 32 bit.
637 FpuCSel32 {
638 rd: Writable<Reg>,
639 rn: Reg,
640 rm: Reg,
641 cond: Cond,
642 },
643 /// FP conditional select, 64 bit.
644 FpuCSel64 {
645 rd: Writable<Reg>,
646 rn: Reg,
647 rm: Reg,
648 cond: Cond,
649 },
650
651 /// Round to integer.
652 FpuRound {
653 op: FpuRoundMode,
654 rd: Writable<Reg>,
655 rn: Reg,
656 },
657
658 /// Move to a vector register from a GPR.
659 MovToVec64 {
660 rd: Writable<Reg>,
661 rn: Reg,
662 },
663
664 /// Move to a GPR from a vector register.
665 MovFromVec64 {
666 rd: Writable<Reg>,
667 rn: Reg,
668 },
669
670 /// A vector ALU op.
671 VecRRR {
672 alu_op: VecALUOp,
673 rd: Writable<Reg>,
674 rn: Reg,
675 rm: Reg,
676 ty: Type,
677 },
678
679 /// Vector two register miscellaneous instruction.
680 VecMisc {
681 op: VecMisc2,
682 rd: Writable<Reg>,
683 rn: Reg,
684 ty: Type,
685 },
686
687 /// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn).
688 MovToNZCV {
689 rn: Reg,
690 },
691
692 /// Move from the NZCV flags (actually a `MRS Xn, NZCV` insn).
693 MovFromNZCV {
694 rd: Writable<Reg>,
695 },
696
697 /// Set a register to 1 if condition, else 0.
698 CondSet {
699 rd: Writable<Reg>,
700 cond: Cond,
701 },
702
703 /// A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation
704 /// of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the
705 /// code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit
706 /// target.
707 Call {
708 info: Box<CallInfo>,
709 },
710 /// A machine indirect-call instruction.
711 CallInd {
712 info: Box<CallIndInfo>,
713 },
714
715 // ---- branches (exactly one must appear at end of BB) ----
716 /// A machine return instruction.
717 Ret,
718
719 /// A placeholder instruction, generating no code, meaning that a function epilogue must be
720 /// inserted there.
721 EpiloguePlaceholder,
722
723 /// An unconditional branch.
724 Jump {
725 dest: BranchTarget,
726 },
727
728 /// A conditional branch. Contains two targets; at emission time, both are emitted, but
729 /// the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
730 /// choice of taken/not_taken (inverting the branch polarity as needed) based on the
731 /// fallthrough at the time of lowering.
732 CondBr {
733 taken: BranchTarget,
734 not_taken: BranchTarget,
735 kind: CondBrKind,
736 },
737
738 /// A one-way conditional branch, invisible to the CFG processing; used *only* as part of
739 /// straight-line sequences in code to be emitted.
740 ///
741 /// In more detail:
742 /// - This branch is lowered to a branch at the machine-code level, but does not end a basic
743 /// block, and does not create edges in the CFG seen by regalloc.
744 /// - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is
745 /// lowered from a single CLIF instruction. For example, certain arithmetic operations may
746 /// use these branches to handle certain conditions, such as overflows, traps, etc.
747 ///
748 /// See, e.g., the lowering of `trapif` (conditional trap) for an example.
749 OneWayCondBr {
750 target: BranchTarget,
751 kind: CondBrKind,
752 },
753
754 /// An indirect branch through a register, augmented with set of all
755 /// possible successors.
756 IndirectBr {
757 rn: Reg,
758 targets: Vec<MachLabel>,
759 },
760
761 /// A "break" instruction, used for e.g. traps and debug breakpoints.
762 Brk,
763
764 /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at
765 /// runtime.
766 Udf {
767 trap_info: (SourceLoc, TrapCode),
768 },
769
770 /// Compute the address (using a PC-relative offset) of a memory location, using the `ADR`
771 /// instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is
772 /// only used for now in fixed lowering sequences with hardcoded offsets. In the future we may
773 /// need full `MemLabel` support.
774 Adr {
775 rd: Writable<Reg>,
776 /// Offset in range -2^20 .. 2^20.
777 off: i32,
778 },
779
780 /// Raw 32-bit word, used for inline constants and jump-table entries.
781 Word4 {
782 data: u32,
783 },
784
785 /// Raw 64-bit word, used for inline constants.
786 Word8 {
787 data: u64,
788 },
789
790 /// Jump-table sequence, as one compound instruction (see note in lower.rs
791 /// for rationale).
792 JTSequence {
793 info: Box<JTSequenceInfo>,
794 ridx: Reg,
795 rtmp1: Writable<Reg>,
796 rtmp2: Writable<Reg>,
797 },
798
799 /// Load an inline constant.
800 LoadConst64 {
801 rd: Writable<Reg>,
802 const_data: u64,
803 },
804
805 /// Load an inline symbol reference.
806 LoadExtName {
807 rd: Writable<Reg>,
808 name: Box<ExternalName>,
809 srcloc: SourceLoc,
810 offset: i64,
811 },
812
813 /// Load address referenced by `mem` into `rd`.
814 LoadAddr {
815 rd: Writable<Reg>,
816 mem: MemArg,
817 },
818
819 /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This
820 /// controls MemArg::NominalSPOffset args are lowered.
821 VirtualSPOffsetAdj {
822 offset: i64,
823 },
824
825 /// Meta-insn, no-op in generated code: emit constant/branch veneer island
826 /// at this point (with a guard jump around it) if less than the needed
827 /// space is available before the next branch deadline. See the `MachBuffer`
828 /// implementation in `machinst/buffer.rs` for the overall algorithm. In
829 /// brief, we retain a set of "pending/unresolved label references" from
830 /// branches as we scan forward through instructions to emit machine code;
831 /// if we notice we're about to go out of range on an unresolved reference,
832 /// we stop, emit a bunch of "veneers" (branches in a form that has a longer
833 /// range, e.g. a 26-bit-offset unconditional jump), and point the original
834 /// label references to those. This is an "island" because it comes in the
835 /// middle of the code.
836 ///
837 /// This meta-instruction is a necessary part of the logic that determines
838 /// where to place islands. Ordinarily, we want to place them between basic
839 /// blocks, so we compute the worst-case size of each block, and emit the
840 /// island before starting a block if we would exceed a deadline before the
841 /// end of the block. However, some sequences (such as an inline jumptable)
842 /// are variable-length and not accounted for by this logic; so these
843 /// lowered sequences include an `EmitIsland` to trigger island generation
844 /// where necessary.
845 EmitIsland {
846 /// The needed space before the next deadline.
847 needed_space: CodeOffset,
848 },
849 }
850
count_zero_half_words(mut value: u64) -> usize851 fn count_zero_half_words(mut value: u64) -> usize {
852 let mut count = 0;
853 for _ in 0..4 {
854 if value & 0xffff == 0 {
855 count += 1;
856 }
857 value >>= 16;
858 }
859
860 count
861 }
862
863 #[test]
inst_size_test()864 fn inst_size_test() {
865 // This test will help with unintentionally growing the size
866 // of the Inst enum.
867 assert_eq!(32, std::mem::size_of::<Inst>());
868 }
869
870 impl Inst {
871 /// Create a move instruction.
mov(to_reg: Writable<Reg>, from_reg: Reg) -> Inst872 pub fn mov(to_reg: Writable<Reg>, from_reg: Reg) -> Inst {
873 assert!(to_reg.to_reg().get_class() == from_reg.get_class());
874 if from_reg.get_class() == RegClass::I64 {
875 Inst::Mov {
876 rd: to_reg,
877 rm: from_reg,
878 }
879 } else if from_reg.get_class() == RegClass::V128 {
880 Inst::FpuMove128 {
881 rd: to_reg,
882 rn: from_reg,
883 }
884 } else {
885 Inst::FpuMove64 {
886 rd: to_reg,
887 rn: from_reg,
888 }
889 }
890 }
891
892 /// Create a 32-bit move instruction.
mov32(to_reg: Writable<Reg>, from_reg: Reg) -> Inst893 pub fn mov32(to_reg: Writable<Reg>, from_reg: Reg) -> Inst {
894 Inst::Mov32 {
895 rd: to_reg,
896 rm: from_reg,
897 }
898 }
899
900 /// Create an instruction that loads a constant, using one of serveral options (MOVZ, MOVN,
901 /// logical immediate, or constant pool).
load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]>902 pub fn load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
903 if let Some(imm) = MoveWideConst::maybe_from_u64(value) {
904 // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ
905 smallvec![Inst::MovZ { rd, imm }]
906 } else if let Some(imm) = MoveWideConst::maybe_from_u64(!value) {
907 // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN
908 smallvec![Inst::MovN { rd, imm }]
909 } else if let Some(imml) = ImmLogic::maybe_from_u64(value, I64) {
910 // Weird logical-instruction immediate in ORI using zero register
911 smallvec![Inst::AluRRImmLogic {
912 alu_op: ALUOp::Orr64,
913 rd,
914 rn: zero_reg(),
915 imml,
916 }]
917 } else {
918 let mut insts = smallvec![];
919
920 // If the number of 0xffff half words is greater than the number of 0x0000 half words
921 // it is more efficient to use `movn` for the first instruction.
922 let first_is_inverted = count_zero_half_words(!value) > count_zero_half_words(value);
923 // Either 0xffff or 0x0000 half words can be skipped, depending on the first
924 // instruction used.
925 let ignored_halfword = if first_is_inverted { 0xffff } else { 0 };
926 let mut first_mov_emitted = false;
927
928 for i in 0..4 {
929 let imm16 = (value >> (16 * i)) & 0xffff;
930 if imm16 != ignored_halfword {
931 if !first_mov_emitted {
932 first_mov_emitted = true;
933 if first_is_inverted {
934 let imm =
935 MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, i * 16)
936 .unwrap();
937 insts.push(Inst::MovN { rd, imm });
938 } else {
939 let imm =
940 MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap();
941 insts.push(Inst::MovZ { rd, imm });
942 }
943 } else {
944 let imm = MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap();
945 insts.push(Inst::MovK { rd, imm });
946 }
947 }
948 }
949
950 assert!(first_mov_emitted);
951
952 insts
953 }
954 }
955
956 /// Create an instruction that loads a 32-bit floating-point constant.
load_fp_constant32(rd: Writable<Reg>, value: f32) -> Inst957 pub fn load_fp_constant32(rd: Writable<Reg>, value: f32) -> Inst {
958 // TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent bits.
959 Inst::LoadFpuConst32 {
960 rd,
961 const_data: value,
962 }
963 }
964
965 /// Create an instruction that loads a 64-bit floating-point constant.
load_fp_constant64(rd: Writable<Reg>, value: f64) -> Inst966 pub fn load_fp_constant64(rd: Writable<Reg>, value: f64) -> Inst {
967 // TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent bits.
968 Inst::LoadFpuConst64 {
969 rd,
970 const_data: value,
971 }
972 }
973
974 /// Create an instruction that loads a 128-bit vector constant.
load_fp_constant128(rd: Writable<Reg>, value: u128) -> Inst975 pub fn load_fp_constant128(rd: Writable<Reg>, value: u128) -> Inst {
976 Inst::LoadFpuConst128 {
977 rd,
978 const_data: value,
979 }
980 }
981 }
982
983 //=============================================================================
984 // Instructions: get_regs
985
memarg_regs(memarg: &MemArg, collector: &mut RegUsageCollector)986 fn memarg_regs(memarg: &MemArg, collector: &mut RegUsageCollector) {
987 match memarg {
988 &MemArg::Unscaled(reg, ..) | &MemArg::UnsignedOffset(reg, ..) => {
989 collector.add_use(reg);
990 }
991 &MemArg::RegReg(r1, r2, ..)
992 | &MemArg::RegScaled(r1, r2, ..)
993 | &MemArg::RegScaledExtended(r1, r2, ..) => {
994 collector.add_use(r1);
995 collector.add_use(r2);
996 }
997 &MemArg::Label(..) => {}
998 &MemArg::PreIndexed(reg, ..) | &MemArg::PostIndexed(reg, ..) => {
999 collector.add_mod(reg);
1000 }
1001 &MemArg::FPOffset(..) => {
1002 collector.add_use(fp_reg());
1003 }
1004 &MemArg::SPOffset(..) | &MemArg::NominalSPOffset(..) => {
1005 collector.add_use(stack_reg());
1006 }
1007 &MemArg::RegOffset(r, ..) => {
1008 collector.add_use(r);
1009 }
1010 }
1011 }
1012
pairmemarg_regs(pairmemarg: &PairMemArg, collector: &mut RegUsageCollector)1013 fn pairmemarg_regs(pairmemarg: &PairMemArg, collector: &mut RegUsageCollector) {
1014 match pairmemarg {
1015 &PairMemArg::SignedOffset(reg, ..) => {
1016 collector.add_use(reg);
1017 }
1018 &PairMemArg::PreIndexed(reg, ..) | &PairMemArg::PostIndexed(reg, ..) => {
1019 collector.add_mod(reg);
1020 }
1021 }
1022 }
1023
aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector)1024 fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
1025 match inst {
1026 &Inst::AluRRR { rd, rn, rm, .. } => {
1027 collector.add_def(rd);
1028 collector.add_use(rn);
1029 collector.add_use(rm);
1030 }
1031 &Inst::AluRRRR { rd, rn, rm, ra, .. } => {
1032 collector.add_def(rd);
1033 collector.add_use(rn);
1034 collector.add_use(rm);
1035 collector.add_use(ra);
1036 }
1037 &Inst::AluRRImm12 { rd, rn, .. } => {
1038 collector.add_def(rd);
1039 collector.add_use(rn);
1040 }
1041 &Inst::AluRRImmLogic { rd, rn, .. } => {
1042 collector.add_def(rd);
1043 collector.add_use(rn);
1044 }
1045 &Inst::AluRRImmShift { rd, rn, .. } => {
1046 collector.add_def(rd);
1047 collector.add_use(rn);
1048 }
1049 &Inst::AluRRRShift { rd, rn, rm, .. } => {
1050 collector.add_def(rd);
1051 collector.add_use(rn);
1052 collector.add_use(rm);
1053 }
1054 &Inst::AluRRRExtend { rd, rn, rm, .. } => {
1055 collector.add_def(rd);
1056 collector.add_use(rn);
1057 collector.add_use(rm);
1058 }
1059 &Inst::BitRR { rd, rn, .. } => {
1060 collector.add_def(rd);
1061 collector.add_use(rn);
1062 }
1063 &Inst::ULoad8 { rd, ref mem, .. }
1064 | &Inst::SLoad8 { rd, ref mem, .. }
1065 | &Inst::ULoad16 { rd, ref mem, .. }
1066 | &Inst::SLoad16 { rd, ref mem, .. }
1067 | &Inst::ULoad32 { rd, ref mem, .. }
1068 | &Inst::SLoad32 { rd, ref mem, .. }
1069 | &Inst::ULoad64 { rd, ref mem, .. } => {
1070 collector.add_def(rd);
1071 memarg_regs(mem, collector);
1072 }
1073 &Inst::Store8 { rd, ref mem, .. }
1074 | &Inst::Store16 { rd, ref mem, .. }
1075 | &Inst::Store32 { rd, ref mem, .. }
1076 | &Inst::Store64 { rd, ref mem, .. } => {
1077 collector.add_use(rd);
1078 memarg_regs(mem, collector);
1079 }
1080 &Inst::StoreP64 {
1081 rt, rt2, ref mem, ..
1082 } => {
1083 collector.add_use(rt);
1084 collector.add_use(rt2);
1085 pairmemarg_regs(mem, collector);
1086 }
1087 &Inst::LoadP64 {
1088 rt, rt2, ref mem, ..
1089 } => {
1090 collector.add_def(rt);
1091 collector.add_def(rt2);
1092 pairmemarg_regs(mem, collector);
1093 }
1094 &Inst::Mov { rd, rm } => {
1095 collector.add_def(rd);
1096 collector.add_use(rm);
1097 }
1098 &Inst::Mov32 { rd, rm } => {
1099 collector.add_def(rd);
1100 collector.add_use(rm);
1101 }
1102 &Inst::MovZ { rd, .. } | &Inst::MovN { rd, .. } => {
1103 collector.add_def(rd);
1104 }
1105 &Inst::MovK { rd, .. } => {
1106 collector.add_mod(rd);
1107 }
1108 &Inst::CSel { rd, rn, rm, .. } => {
1109 collector.add_def(rd);
1110 collector.add_use(rn);
1111 collector.add_use(rm);
1112 }
1113 &Inst::CSet { rd, .. } => {
1114 collector.add_def(rd);
1115 }
1116 &Inst::CCmpImm { rn, .. } => {
1117 collector.add_use(rn);
1118 }
1119 &Inst::FpuMove64 { rd, rn } => {
1120 collector.add_def(rd);
1121 collector.add_use(rn);
1122 }
1123 &Inst::FpuMove128 { rd, rn } => {
1124 collector.add_def(rd);
1125 collector.add_use(rn);
1126 }
1127 &Inst::FpuRR { rd, rn, .. } => {
1128 collector.add_def(rd);
1129 collector.add_use(rn);
1130 }
1131 &Inst::FpuRRR { rd, rn, rm, .. } => {
1132 collector.add_def(rd);
1133 collector.add_use(rn);
1134 collector.add_use(rm);
1135 }
1136 &Inst::FpuRRI { fpu_op, rd, rn, .. } => {
1137 match fpu_op {
1138 FPUOpRI::UShr32(..) | FPUOpRI::UShr64(..) => collector.add_def(rd),
1139 FPUOpRI::Sli32(..) | FPUOpRI::Sli64(..) => collector.add_mod(rd),
1140 }
1141 collector.add_use(rn);
1142 }
1143 &Inst::FpuRRRR { rd, rn, rm, ra, .. } => {
1144 collector.add_def(rd);
1145 collector.add_use(rn);
1146 collector.add_use(rm);
1147 collector.add_use(ra);
1148 }
1149 &Inst::VecMisc { rd, rn, .. } => {
1150 collector.add_def(rd);
1151 collector.add_use(rn);
1152 }
1153 &Inst::FpuCmp32 { rn, rm } | &Inst::FpuCmp64 { rn, rm } => {
1154 collector.add_use(rn);
1155 collector.add_use(rm);
1156 }
1157 &Inst::FpuLoad32 { rd, ref mem, .. } => {
1158 collector.add_def(rd);
1159 memarg_regs(mem, collector);
1160 }
1161 &Inst::FpuLoad64 { rd, ref mem, .. } => {
1162 collector.add_def(rd);
1163 memarg_regs(mem, collector);
1164 }
1165 &Inst::FpuLoad128 { rd, ref mem, .. } => {
1166 collector.add_def(rd);
1167 memarg_regs(mem, collector);
1168 }
1169 &Inst::FpuStore32 { rd, ref mem, .. } => {
1170 collector.add_use(rd);
1171 memarg_regs(mem, collector);
1172 }
1173 &Inst::FpuStore64 { rd, ref mem, .. } => {
1174 collector.add_use(rd);
1175 memarg_regs(mem, collector);
1176 }
1177 &Inst::FpuStore128 { rd, ref mem, .. } => {
1178 collector.add_use(rd);
1179 memarg_regs(mem, collector);
1180 }
1181 &Inst::LoadFpuConst32 { rd, .. }
1182 | &Inst::LoadFpuConst64 { rd, .. }
1183 | &Inst::LoadFpuConst128 { rd, .. } => {
1184 collector.add_def(rd);
1185 }
1186 &Inst::FpuToInt { rd, rn, .. } => {
1187 collector.add_def(rd);
1188 collector.add_use(rn);
1189 }
1190 &Inst::IntToFpu { rd, rn, .. } => {
1191 collector.add_def(rd);
1192 collector.add_use(rn);
1193 }
1194 &Inst::FpuCSel32 { rd, rn, rm, .. } | &Inst::FpuCSel64 { rd, rn, rm, .. } => {
1195 collector.add_def(rd);
1196 collector.add_use(rn);
1197 collector.add_use(rm);
1198 }
1199 &Inst::FpuRound { rd, rn, .. } => {
1200 collector.add_def(rd);
1201 collector.add_use(rn);
1202 }
1203 &Inst::MovToVec64 { rd, rn } => {
1204 collector.add_def(rd);
1205 collector.add_use(rn);
1206 }
1207 &Inst::MovFromVec64 { rd, rn } => {
1208 collector.add_def(rd);
1209 collector.add_use(rn);
1210 }
1211 &Inst::VecRRR { rd, rn, rm, .. } => {
1212 collector.add_def(rd);
1213 collector.add_use(rn);
1214 collector.add_use(rm);
1215 }
1216 &Inst::MovToNZCV { rn } => {
1217 collector.add_use(rn);
1218 }
1219 &Inst::MovFromNZCV { rd } => {
1220 collector.add_def(rd);
1221 }
1222 &Inst::CondSet { rd, .. } => {
1223 collector.add_def(rd);
1224 }
1225 &Inst::Extend { rd, rn, .. } => {
1226 collector.add_def(rd);
1227 collector.add_use(rn);
1228 }
1229 &Inst::Jump { .. } | &Inst::Ret | &Inst::EpiloguePlaceholder => {}
1230 &Inst::Call { ref info } => {
1231 collector.add_uses(&*info.uses);
1232 collector.add_defs(&*info.defs);
1233 }
1234 &Inst::CallInd { ref info } => {
1235 collector.add_uses(&*info.uses);
1236 collector.add_defs(&*info.defs);
1237 collector.add_use(info.rn);
1238 }
1239 &Inst::CondBr { ref kind, .. } | &Inst::OneWayCondBr { ref kind, .. } => match kind {
1240 CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
1241 collector.add_use(*rt);
1242 }
1243 CondBrKind::Cond(_) => {}
1244 },
1245 &Inst::IndirectBr { rn, .. } => {
1246 collector.add_use(rn);
1247 }
1248 &Inst::Nop0 | Inst::Nop4 => {}
1249 &Inst::Brk => {}
1250 &Inst::Udf { .. } => {}
1251 &Inst::Adr { rd, .. } => {
1252 collector.add_def(rd);
1253 }
1254 &Inst::Word4 { .. } | &Inst::Word8 { .. } => {}
1255 &Inst::JTSequence {
1256 ridx, rtmp1, rtmp2, ..
1257 } => {
1258 collector.add_use(ridx);
1259 collector.add_def(rtmp1);
1260 collector.add_def(rtmp2);
1261 }
1262 &Inst::LoadConst64 { rd, .. } | &Inst::LoadExtName { rd, .. } => {
1263 collector.add_def(rd);
1264 }
1265 &Inst::LoadAddr { rd, mem: _ } => {
1266 collector.add_def(rd);
1267 }
1268 &Inst::VirtualSPOffsetAdj { .. } => {}
1269 &Inst::EmitIsland { .. } => {}
1270 }
1271 }
1272
1273 //=============================================================================
1274 // Instructions: map_regs
1275
aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM)1276 fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
1277 fn map_use<RUM: RegUsageMapper>(m: &RUM, r: &mut Reg) {
1278 if r.is_virtual() {
1279 let new = m.get_use(r.to_virtual_reg()).unwrap().to_reg();
1280 *r = new;
1281 }
1282 }
1283
1284 fn map_def<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
1285 if r.to_reg().is_virtual() {
1286 let new = m.get_def(r.to_reg().to_virtual_reg()).unwrap().to_reg();
1287 *r = Writable::from_reg(new);
1288 }
1289 }
1290
1291 fn map_mod<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
1292 if r.to_reg().is_virtual() {
1293 let new = m.get_mod(r.to_reg().to_virtual_reg()).unwrap().to_reg();
1294 *r = Writable::from_reg(new);
1295 }
1296 }
1297
1298 fn map_mem<RUM: RegUsageMapper>(m: &RUM, mem: &mut MemArg) {
1299 // N.B.: we take only the pre-map here, but this is OK because the
1300 // only addressing modes that update registers (pre/post-increment on
1301 // AArch64) both read and write registers, so they are "mods" rather
1302 // than "defs", so must be the same in both the pre- and post-map.
1303 match mem {
1304 &mut MemArg::Unscaled(ref mut reg, ..) => map_use(m, reg),
1305 &mut MemArg::UnsignedOffset(ref mut reg, ..) => map_use(m, reg),
1306 &mut MemArg::RegReg(ref mut r1, ref mut r2) => {
1307 map_use(m, r1);
1308 map_use(m, r2);
1309 }
1310 &mut MemArg::RegScaled(ref mut r1, ref mut r2, ..) => {
1311 map_use(m, r1);
1312 map_use(m, r2);
1313 }
1314 &mut MemArg::RegScaledExtended(ref mut r1, ref mut r2, ..) => {
1315 map_use(m, r1);
1316 map_use(m, r2);
1317 }
1318 &mut MemArg::Label(..) => {}
1319 &mut MemArg::PreIndexed(ref mut r, ..) => map_mod(m, r),
1320 &mut MemArg::PostIndexed(ref mut r, ..) => map_mod(m, r),
1321 &mut MemArg::FPOffset(..)
1322 | &mut MemArg::SPOffset(..)
1323 | &mut MemArg::NominalSPOffset(..) => {}
1324 &mut MemArg::RegOffset(ref mut r, ..) => map_use(m, r),
1325 };
1326 }
1327
1328 fn map_pairmem<RUM: RegUsageMapper>(m: &RUM, mem: &mut PairMemArg) {
1329 match mem {
1330 &mut PairMemArg::SignedOffset(ref mut reg, ..) => map_use(m, reg),
1331 &mut PairMemArg::PreIndexed(ref mut reg, ..) => map_def(m, reg),
1332 &mut PairMemArg::PostIndexed(ref mut reg, ..) => map_def(m, reg),
1333 }
1334 }
1335
1336 fn map_br<RUM: RegUsageMapper>(m: &RUM, br: &mut CondBrKind) {
1337 match br {
1338 &mut CondBrKind::Zero(ref mut reg) => map_use(m, reg),
1339 &mut CondBrKind::NotZero(ref mut reg) => map_use(m, reg),
1340 &mut CondBrKind::Cond(..) => {}
1341 };
1342 }
1343
1344 match inst {
1345 &mut Inst::AluRRR {
1346 ref mut rd,
1347 ref mut rn,
1348 ref mut rm,
1349 ..
1350 } => {
1351 map_def(mapper, rd);
1352 map_use(mapper, rn);
1353 map_use(mapper, rm);
1354 }
1355 &mut Inst::AluRRRR {
1356 ref mut rd,
1357 ref mut rn,
1358 ref mut rm,
1359 ref mut ra,
1360 ..
1361 } => {
1362 map_def(mapper, rd);
1363 map_use(mapper, rn);
1364 map_use(mapper, rm);
1365 map_use(mapper, ra);
1366 }
1367 &mut Inst::AluRRImm12 {
1368 ref mut rd,
1369 ref mut rn,
1370 ..
1371 } => {
1372 map_def(mapper, rd);
1373 map_use(mapper, rn);
1374 }
1375 &mut Inst::AluRRImmLogic {
1376 ref mut rd,
1377 ref mut rn,
1378 ..
1379 } => {
1380 map_def(mapper, rd);
1381 map_use(mapper, rn);
1382 }
1383 &mut Inst::AluRRImmShift {
1384 ref mut rd,
1385 ref mut rn,
1386 ..
1387 } => {
1388 map_def(mapper, rd);
1389 map_use(mapper, rn);
1390 }
1391 &mut Inst::AluRRRShift {
1392 ref mut rd,
1393 ref mut rn,
1394 ref mut rm,
1395 ..
1396 } => {
1397 map_def(mapper, rd);
1398 map_use(mapper, rn);
1399 map_use(mapper, rm);
1400 }
1401 &mut Inst::AluRRRExtend {
1402 ref mut rd,
1403 ref mut rn,
1404 ref mut rm,
1405 ..
1406 } => {
1407 map_def(mapper, rd);
1408 map_use(mapper, rn);
1409 map_use(mapper, rm);
1410 }
1411 &mut Inst::BitRR {
1412 ref mut rd,
1413 ref mut rn,
1414 ..
1415 } => {
1416 map_def(mapper, rd);
1417 map_use(mapper, rn);
1418 }
1419 &mut Inst::ULoad8 {
1420 ref mut rd,
1421 ref mut mem,
1422 ..
1423 } => {
1424 map_def(mapper, rd);
1425 map_mem(mapper, mem);
1426 }
1427 &mut Inst::SLoad8 {
1428 ref mut rd,
1429 ref mut mem,
1430 ..
1431 } => {
1432 map_def(mapper, rd);
1433 map_mem(mapper, mem);
1434 }
1435 &mut Inst::ULoad16 {
1436 ref mut rd,
1437 ref mut mem,
1438 ..
1439 } => {
1440 map_def(mapper, rd);
1441 map_mem(mapper, mem);
1442 }
1443 &mut Inst::SLoad16 {
1444 ref mut rd,
1445 ref mut mem,
1446 ..
1447 } => {
1448 map_def(mapper, rd);
1449 map_mem(mapper, mem);
1450 }
1451 &mut Inst::ULoad32 {
1452 ref mut rd,
1453 ref mut mem,
1454 ..
1455 } => {
1456 map_def(mapper, rd);
1457 map_mem(mapper, mem);
1458 }
1459 &mut Inst::SLoad32 {
1460 ref mut rd,
1461 ref mut mem,
1462 ..
1463 } => {
1464 map_def(mapper, rd);
1465 map_mem(mapper, mem);
1466 }
1467
1468 &mut Inst::ULoad64 {
1469 ref mut rd,
1470 ref mut mem,
1471 ..
1472 } => {
1473 map_def(mapper, rd);
1474 map_mem(mapper, mem);
1475 }
1476 &mut Inst::Store8 {
1477 ref mut rd,
1478 ref mut mem,
1479 ..
1480 } => {
1481 map_use(mapper, rd);
1482 map_mem(mapper, mem);
1483 }
1484 &mut Inst::Store16 {
1485 ref mut rd,
1486 ref mut mem,
1487 ..
1488 } => {
1489 map_use(mapper, rd);
1490 map_mem(mapper, mem);
1491 }
1492 &mut Inst::Store32 {
1493 ref mut rd,
1494 ref mut mem,
1495 ..
1496 } => {
1497 map_use(mapper, rd);
1498 map_mem(mapper, mem);
1499 }
1500 &mut Inst::Store64 {
1501 ref mut rd,
1502 ref mut mem,
1503 ..
1504 } => {
1505 map_use(mapper, rd);
1506 map_mem(mapper, mem);
1507 }
1508
1509 &mut Inst::StoreP64 {
1510 ref mut rt,
1511 ref mut rt2,
1512 ref mut mem,
1513 } => {
1514 map_use(mapper, rt);
1515 map_use(mapper, rt2);
1516 map_pairmem(mapper, mem);
1517 }
1518 &mut Inst::LoadP64 {
1519 ref mut rt,
1520 ref mut rt2,
1521 ref mut mem,
1522 } => {
1523 map_def(mapper, rt);
1524 map_def(mapper, rt2);
1525 map_pairmem(mapper, mem);
1526 }
1527 &mut Inst::Mov {
1528 ref mut rd,
1529 ref mut rm,
1530 } => {
1531 map_def(mapper, rd);
1532 map_use(mapper, rm);
1533 }
1534 &mut Inst::Mov32 {
1535 ref mut rd,
1536 ref mut rm,
1537 } => {
1538 map_def(mapper, rd);
1539 map_use(mapper, rm);
1540 }
1541 &mut Inst::MovZ { ref mut rd, .. } => {
1542 map_def(mapper, rd);
1543 }
1544 &mut Inst::MovN { ref mut rd, .. } => {
1545 map_def(mapper, rd);
1546 }
1547 &mut Inst::MovK { ref mut rd, .. } => {
1548 map_def(mapper, rd);
1549 }
1550 &mut Inst::CSel {
1551 ref mut rd,
1552 ref mut rn,
1553 ref mut rm,
1554 ..
1555 } => {
1556 map_def(mapper, rd);
1557 map_use(mapper, rn);
1558 map_use(mapper, rm);
1559 }
1560 &mut Inst::CSet { ref mut rd, .. } => {
1561 map_def(mapper, rd);
1562 }
1563 &mut Inst::CCmpImm { ref mut rn, .. } => {
1564 map_use(mapper, rn);
1565 }
1566 &mut Inst::FpuMove64 {
1567 ref mut rd,
1568 ref mut rn,
1569 } => {
1570 map_def(mapper, rd);
1571 map_use(mapper, rn);
1572 }
1573 &mut Inst::FpuMove128 {
1574 ref mut rd,
1575 ref mut rn,
1576 } => {
1577 map_def(mapper, rd);
1578 map_use(mapper, rn);
1579 }
1580 &mut Inst::FpuRR {
1581 ref mut rd,
1582 ref mut rn,
1583 ..
1584 } => {
1585 map_def(mapper, rd);
1586 map_use(mapper, rn);
1587 }
1588 &mut Inst::FpuRRR {
1589 ref mut rd,
1590 ref mut rn,
1591 ref mut rm,
1592 ..
1593 } => {
1594 map_def(mapper, rd);
1595 map_use(mapper, rn);
1596 map_use(mapper, rm);
1597 }
1598 &mut Inst::FpuRRI {
1599 ref mut rd,
1600 ref mut rn,
1601 ..
1602 } => {
1603 map_def(mapper, rd);
1604 map_use(mapper, rn);
1605 }
1606 &mut Inst::FpuRRRR {
1607 ref mut rd,
1608 ref mut rn,
1609 ref mut rm,
1610 ref mut ra,
1611 ..
1612 } => {
1613 map_def(mapper, rd);
1614 map_use(mapper, rn);
1615 map_use(mapper, rm);
1616 map_use(mapper, ra);
1617 }
1618 &mut Inst::VecMisc {
1619 ref mut rd,
1620 ref mut rn,
1621 ..
1622 } => {
1623 map_def(mapper, rd);
1624 map_use(mapper, rn);
1625 }
1626 &mut Inst::FpuCmp32 {
1627 ref mut rn,
1628 ref mut rm,
1629 } => {
1630 map_use(mapper, rn);
1631 map_use(mapper, rm);
1632 }
1633 &mut Inst::FpuCmp64 {
1634 ref mut rn,
1635 ref mut rm,
1636 } => {
1637 map_use(mapper, rn);
1638 map_use(mapper, rm);
1639 }
1640 &mut Inst::FpuLoad32 {
1641 ref mut rd,
1642 ref mut mem,
1643 ..
1644 } => {
1645 map_def(mapper, rd);
1646 map_mem(mapper, mem);
1647 }
1648 &mut Inst::FpuLoad64 {
1649 ref mut rd,
1650 ref mut mem,
1651 ..
1652 } => {
1653 map_def(mapper, rd);
1654 map_mem(mapper, mem);
1655 }
1656 &mut Inst::FpuLoad128 {
1657 ref mut rd,
1658 ref mut mem,
1659 ..
1660 } => {
1661 map_def(mapper, rd);
1662 map_mem(mapper, mem);
1663 }
1664 &mut Inst::FpuStore32 {
1665 ref mut rd,
1666 ref mut mem,
1667 ..
1668 } => {
1669 map_use(mapper, rd);
1670 map_mem(mapper, mem);
1671 }
1672 &mut Inst::FpuStore64 {
1673 ref mut rd,
1674 ref mut mem,
1675 ..
1676 } => {
1677 map_use(mapper, rd);
1678 map_mem(mapper, mem);
1679 }
1680 &mut Inst::FpuStore128 {
1681 ref mut rd,
1682 ref mut mem,
1683 ..
1684 } => {
1685 map_use(mapper, rd);
1686 map_mem(mapper, mem);
1687 }
1688 &mut Inst::LoadFpuConst32 { ref mut rd, .. } => {
1689 map_def(mapper, rd);
1690 }
1691 &mut Inst::LoadFpuConst64 { ref mut rd, .. } => {
1692 map_def(mapper, rd);
1693 }
1694 &mut Inst::LoadFpuConst128 { ref mut rd, .. } => {
1695 map_def(mapper, rd);
1696 }
1697 &mut Inst::FpuToInt {
1698 ref mut rd,
1699 ref mut rn,
1700 ..
1701 } => {
1702 map_def(mapper, rd);
1703 map_use(mapper, rn);
1704 }
1705 &mut Inst::IntToFpu {
1706 ref mut rd,
1707 ref mut rn,
1708 ..
1709 } => {
1710 map_def(mapper, rd);
1711 map_use(mapper, rn);
1712 }
1713 &mut Inst::FpuCSel32 {
1714 ref mut rd,
1715 ref mut rn,
1716 ref mut rm,
1717 ..
1718 } => {
1719 map_def(mapper, rd);
1720 map_use(mapper, rn);
1721 map_use(mapper, rm);
1722 }
1723 &mut Inst::FpuCSel64 {
1724 ref mut rd,
1725 ref mut rn,
1726 ref mut rm,
1727 ..
1728 } => {
1729 map_def(mapper, rd);
1730 map_use(mapper, rn);
1731 map_use(mapper, rm);
1732 }
1733 &mut Inst::FpuRound {
1734 ref mut rd,
1735 ref mut rn,
1736 ..
1737 } => {
1738 map_def(mapper, rd);
1739 map_use(mapper, rn);
1740 }
1741 &mut Inst::MovToVec64 {
1742 ref mut rd,
1743 ref mut rn,
1744 } => {
1745 map_def(mapper, rd);
1746 map_use(mapper, rn);
1747 }
1748 &mut Inst::MovFromVec64 {
1749 ref mut rd,
1750 ref mut rn,
1751 } => {
1752 map_def(mapper, rd);
1753 map_use(mapper, rn);
1754 }
1755 &mut Inst::VecRRR {
1756 ref mut rd,
1757 ref mut rn,
1758 ref mut rm,
1759 ..
1760 } => {
1761 map_def(mapper, rd);
1762 map_use(mapper, rn);
1763 map_use(mapper, rm);
1764 }
1765 &mut Inst::MovToNZCV { ref mut rn } => {
1766 map_use(mapper, rn);
1767 }
1768 &mut Inst::MovFromNZCV { ref mut rd } => {
1769 map_def(mapper, rd);
1770 }
1771 &mut Inst::CondSet { ref mut rd, .. } => {
1772 map_def(mapper, rd);
1773 }
1774 &mut Inst::Extend {
1775 ref mut rd,
1776 ref mut rn,
1777 ..
1778 } => {
1779 map_def(mapper, rd);
1780 map_use(mapper, rn);
1781 }
1782 &mut Inst::Jump { .. } => {}
1783 &mut Inst::Call { ref mut info } => {
1784 for r in info.uses.iter_mut() {
1785 map_use(mapper, r);
1786 }
1787 for r in info.defs.iter_mut() {
1788 map_def(mapper, r);
1789 }
1790 }
1791 &mut Inst::Ret | &mut Inst::EpiloguePlaceholder => {}
1792 &mut Inst::CallInd { ref mut info, .. } => {
1793 for r in info.uses.iter_mut() {
1794 map_use(mapper, r);
1795 }
1796 for r in info.defs.iter_mut() {
1797 map_def(mapper, r);
1798 }
1799 map_use(mapper, &mut info.rn);
1800 }
1801 &mut Inst::CondBr { ref mut kind, .. } | &mut Inst::OneWayCondBr { ref mut kind, .. } => {
1802 map_br(mapper, kind);
1803 }
1804 &mut Inst::IndirectBr { ref mut rn, .. } => {
1805 map_use(mapper, rn);
1806 }
1807 &mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {}
1808 &mut Inst::Adr { ref mut rd, .. } => {
1809 map_def(mapper, rd);
1810 }
1811 &mut Inst::Word4 { .. } | &mut Inst::Word8 { .. } => {}
1812 &mut Inst::JTSequence {
1813 ref mut ridx,
1814 ref mut rtmp1,
1815 ref mut rtmp2,
1816 ..
1817 } => {
1818 map_use(mapper, ridx);
1819 map_def(mapper, rtmp1);
1820 map_def(mapper, rtmp2);
1821 }
1822 &mut Inst::LoadConst64 { ref mut rd, .. } => {
1823 map_def(mapper, rd);
1824 }
1825 &mut Inst::LoadExtName { ref mut rd, .. } => {
1826 map_def(mapper, rd);
1827 }
1828 &mut Inst::LoadAddr {
1829 ref mut rd,
1830 ref mut mem,
1831 } => {
1832 map_def(mapper, rd);
1833 map_mem(mapper, mem);
1834 }
1835 &mut Inst::VirtualSPOffsetAdj { .. } => {}
1836 &mut Inst::EmitIsland { .. } => {}
1837 }
1838 }
1839
1840 //=============================================================================
1841 // Instructions: misc functions and external interface
1842
1843 impl MachInst for Inst {
1844 type LabelUse = LabelUse;
1845
get_regs(&self, collector: &mut RegUsageCollector)1846 fn get_regs(&self, collector: &mut RegUsageCollector) {
1847 aarch64_get_regs(self, collector)
1848 }
1849
map_regs<RUM: RegUsageMapper>(&mut self, mapper: &RUM)1850 fn map_regs<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
1851 aarch64_map_regs(self, mapper);
1852 }
1853
is_move(&self) -> Option<(Writable<Reg>, Reg)>1854 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
1855 match self {
1856 &Inst::Mov { rd, rm } => Some((rd, rm)),
1857 &Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
1858 &Inst::FpuMove128 { rd, rn } => Some((rd, rn)),
1859 _ => None,
1860 }
1861 }
1862
is_epilogue_placeholder(&self) -> bool1863 fn is_epilogue_placeholder(&self) -> bool {
1864 if let Inst::EpiloguePlaceholder = self {
1865 true
1866 } else {
1867 false
1868 }
1869 }
1870
is_term<'a>(&'a self) -> MachTerminator<'a>1871 fn is_term<'a>(&'a self) -> MachTerminator<'a> {
1872 match self {
1873 &Inst::Ret | &Inst::EpiloguePlaceholder => MachTerminator::Ret,
1874 &Inst::Jump { dest } => MachTerminator::Uncond(dest.as_label().unwrap()),
1875 &Inst::CondBr {
1876 taken, not_taken, ..
1877 } => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()),
1878 &Inst::OneWayCondBr { .. } => {
1879 // Explicitly invisible to CFG processing.
1880 MachTerminator::None
1881 }
1882 &Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
1883 &Inst::JTSequence { ref info, .. } => {
1884 MachTerminator::Indirect(&info.targets_for_term[..])
1885 }
1886 _ => MachTerminator::None,
1887 }
1888 }
1889
gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst1890 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
1891 assert!(ty.bits() <= 128);
1892 Inst::mov(to_reg, from_reg)
1893 }
1894
gen_constant(to_reg: Writable<Reg>, value: u64, ty: Type) -> SmallVec<[Inst; 4]>1895 fn gen_constant(to_reg: Writable<Reg>, value: u64, ty: Type) -> SmallVec<[Inst; 4]> {
1896 if ty == F64 {
1897 let mut ret = SmallVec::new();
1898 ret.push(Inst::load_fp_constant64(to_reg, f64::from_bits(value)));
1899 ret
1900 } else if ty == F32 {
1901 let mut ret = SmallVec::new();
1902 ret.push(Inst::load_fp_constant32(
1903 to_reg,
1904 f32::from_bits(value as u32),
1905 ));
1906 ret
1907 } else {
1908 // Must be an integer type.
1909 debug_assert!(
1910 ty == B1
1911 || ty == I8
1912 || ty == B8
1913 || ty == I16
1914 || ty == B16
1915 || ty == I32
1916 || ty == B32
1917 || ty == I64
1918 || ty == B64
1919 );
1920 Inst::load_constant(to_reg, value)
1921 }
1922 }
1923
gen_zero_len_nop() -> Inst1924 fn gen_zero_len_nop() -> Inst {
1925 Inst::Nop0
1926 }
1927
gen_nop(preferred_size: usize) -> Inst1928 fn gen_nop(preferred_size: usize) -> Inst {
1929 // We can't give a NOP (or any insn) < 4 bytes.
1930 assert!(preferred_size >= 4);
1931 Inst::Nop4
1932 }
1933
maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst>1934 fn maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst> {
1935 None
1936 }
1937
rc_for_type(ty: Type) -> CodegenResult<RegClass>1938 fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
1939 match ty {
1940 I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64),
1941 F32 | F64 => Ok(RegClass::V128),
1942 IFLAGS | FFLAGS => Ok(RegClass::I64),
1943 I8X16 => Ok(RegClass::V128),
1944 B8X16 => Ok(RegClass::V128),
1945 _ => Err(CodegenError::Unsupported(format!(
1946 "Unexpected SSA-value type: {}",
1947 ty
1948 ))),
1949 }
1950 }
1951
gen_jump(target: MachLabel) -> Inst1952 fn gen_jump(target: MachLabel) -> Inst {
1953 Inst::Jump {
1954 dest: BranchTarget::Label(target),
1955 }
1956 }
1957
reg_universe(flags: &settings::Flags) -> RealRegUniverse1958 fn reg_universe(flags: &settings::Flags) -> RealRegUniverse {
1959 create_reg_universe(flags)
1960 }
1961
worst_case_size() -> CodeOffset1962 fn worst_case_size() -> CodeOffset {
1963 // The maximum size, in bytes, of any `Inst`'s emitted code. We have at least one case of
1964 // an 8-instruction sequence (saturating int-to-float conversions) with three embedded
1965 // 64-bit f64 constants.
1966 //
1967 // Note that inline jump-tables handle island/pool insertion separately, so we do not need
1968 // to account for them here (otherwise the worst case would be 2^31 * 4, clearly not
1969 // feasible for other reasons).
1970 44
1971 }
1972 }
1973
1974 //=============================================================================
1975 // Pretty-printing of instructions.
1976
mem_finalize_for_show(mem: &MemArg, mb_rru: Option<&RealRegUniverse>) -> (String, MemArg)1977 fn mem_finalize_for_show(mem: &MemArg, mb_rru: Option<&RealRegUniverse>) -> (String, MemArg) {
1978 let (mem_insts, mem) = mem_finalize(0, mem, &mut Default::default());
1979 let mut mem_str = mem_insts
1980 .into_iter()
1981 .map(|inst| inst.show_rru(mb_rru))
1982 .collect::<Vec<_>>()
1983 .join(" ; ");
1984 if !mem_str.is_empty() {
1985 mem_str += " ; ";
1986 }
1987
1988 (mem_str, mem)
1989 }
1990
1991 impl ShowWithRRU for Inst {
show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String1992 fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
1993 fn op_name_size(alu_op: ALUOp) -> (&'static str, InstSize) {
1994 match alu_op {
1995 ALUOp::Add32 => ("add", InstSize::Size32),
1996 ALUOp::Add64 => ("add", InstSize::Size64),
1997 ALUOp::Sub32 => ("sub", InstSize::Size32),
1998 ALUOp::Sub64 => ("sub", InstSize::Size64),
1999 ALUOp::Orr32 => ("orr", InstSize::Size32),
2000 ALUOp::Orr64 => ("orr", InstSize::Size64),
2001 ALUOp::And32 => ("and", InstSize::Size32),
2002 ALUOp::And64 => ("and", InstSize::Size64),
2003 ALUOp::Eor32 => ("eor", InstSize::Size32),
2004 ALUOp::Eor64 => ("eor", InstSize::Size64),
2005 ALUOp::AddS32 => ("adds", InstSize::Size32),
2006 ALUOp::AddS64 => ("adds", InstSize::Size64),
2007 ALUOp::SubS32 => ("subs", InstSize::Size32),
2008 ALUOp::SubS64 => ("subs", InstSize::Size64),
2009 ALUOp::SubS64XR => ("subs", InstSize::Size64),
2010 ALUOp::MAdd32 => ("madd", InstSize::Size32),
2011 ALUOp::MAdd64 => ("madd", InstSize::Size64),
2012 ALUOp::MSub32 => ("msub", InstSize::Size32),
2013 ALUOp::MSub64 => ("msub", InstSize::Size64),
2014 ALUOp::SMulH => ("smulh", InstSize::Size64),
2015 ALUOp::UMulH => ("umulh", InstSize::Size64),
2016 ALUOp::SDiv64 => ("sdiv", InstSize::Size64),
2017 ALUOp::UDiv64 => ("udiv", InstSize::Size64),
2018 ALUOp::AndNot32 => ("bic", InstSize::Size32),
2019 ALUOp::AndNot64 => ("bic", InstSize::Size64),
2020 ALUOp::OrrNot32 => ("orn", InstSize::Size32),
2021 ALUOp::OrrNot64 => ("orn", InstSize::Size64),
2022 ALUOp::EorNot32 => ("eon", InstSize::Size32),
2023 ALUOp::EorNot64 => ("eon", InstSize::Size64),
2024 ALUOp::RotR32 => ("ror", InstSize::Size32),
2025 ALUOp::RotR64 => ("ror", InstSize::Size64),
2026 ALUOp::Lsr32 => ("lsr", InstSize::Size32),
2027 ALUOp::Lsr64 => ("lsr", InstSize::Size64),
2028 ALUOp::Asr32 => ("asr", InstSize::Size32),
2029 ALUOp::Asr64 => ("asr", InstSize::Size64),
2030 ALUOp::Lsl32 => ("lsl", InstSize::Size32),
2031 ALUOp::Lsl64 => ("lsl", InstSize::Size64),
2032 }
2033 }
2034
2035 match self {
2036 &Inst::Nop0 => "nop-zero-len".to_string(),
2037 &Inst::Nop4 => "nop".to_string(),
2038 &Inst::AluRRR { alu_op, rd, rn, rm } => {
2039 let (op, size) = op_name_size(alu_op);
2040 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2041 let rn = show_ireg_sized(rn, mb_rru, size);
2042 let rm = show_ireg_sized(rm, mb_rru, size);
2043 format!("{} {}, {}, {}", op, rd, rn, rm)
2044 }
2045 &Inst::AluRRRR {
2046 alu_op,
2047 rd,
2048 rn,
2049 rm,
2050 ra,
2051 } => {
2052 let (op, size) = op_name_size(alu_op);
2053 let four_args = alu_op != ALUOp::SMulH && alu_op != ALUOp::UMulH;
2054 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2055 let rn = show_ireg_sized(rn, mb_rru, size);
2056 let rm = show_ireg_sized(rm, mb_rru, size);
2057 let ra = show_ireg_sized(ra, mb_rru, size);
2058 if four_args {
2059 format!("{} {}, {}, {}, {}", op, rd, rn, rm, ra)
2060 } else {
2061 // smulh and umulh have Ra "hard-wired" to the zero register
2062 // and the canonical assembly form has only three regs.
2063 format!("{} {}, {}, {}", op, rd, rn, rm)
2064 }
2065 }
2066 &Inst::AluRRImm12 {
2067 alu_op,
2068 rd,
2069 rn,
2070 ref imm12,
2071 } => {
2072 let (op, size) = op_name_size(alu_op);
2073 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2074 let rn = show_ireg_sized(rn, mb_rru, size);
2075
2076 if imm12.bits == 0 && alu_op == ALUOp::Add64 {
2077 // special-case MOV (used for moving into SP).
2078 format!("mov {}, {}", rd, rn)
2079 } else {
2080 let imm12 = imm12.show_rru(mb_rru);
2081 format!("{} {}, {}, {}", op, rd, rn, imm12)
2082 }
2083 }
2084 &Inst::AluRRImmLogic {
2085 alu_op,
2086 rd,
2087 rn,
2088 ref imml,
2089 } => {
2090 let (op, size) = op_name_size(alu_op);
2091 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2092 let rn = show_ireg_sized(rn, mb_rru, size);
2093 let imml = imml.show_rru(mb_rru);
2094 format!("{} {}, {}, {}", op, rd, rn, imml)
2095 }
2096 &Inst::AluRRImmShift {
2097 alu_op,
2098 rd,
2099 rn,
2100 ref immshift,
2101 } => {
2102 let (op, size) = op_name_size(alu_op);
2103 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2104 let rn = show_ireg_sized(rn, mb_rru, size);
2105 let immshift = immshift.show_rru(mb_rru);
2106 format!("{} {}, {}, {}", op, rd, rn, immshift)
2107 }
2108 &Inst::AluRRRShift {
2109 alu_op,
2110 rd,
2111 rn,
2112 rm,
2113 ref shiftop,
2114 } => {
2115 let (op, size) = op_name_size(alu_op);
2116 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2117 let rn = show_ireg_sized(rn, mb_rru, size);
2118 let rm = show_ireg_sized(rm, mb_rru, size);
2119 let shiftop = shiftop.show_rru(mb_rru);
2120 format!("{} {}, {}, {}, {}", op, rd, rn, rm, shiftop)
2121 }
2122 &Inst::AluRRRExtend {
2123 alu_op,
2124 rd,
2125 rn,
2126 rm,
2127 ref extendop,
2128 } => {
2129 let (op, size) = op_name_size(alu_op);
2130 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2131 let rn = show_ireg_sized(rn, mb_rru, size);
2132 let rm = show_ireg_sized(rm, mb_rru, size);
2133 let extendop = extendop.show_rru(mb_rru);
2134 format!("{} {}, {}, {}, {}", op, rd, rn, rm, extendop)
2135 }
2136 &Inst::BitRR { op, rd, rn } => {
2137 let size = op.inst_size();
2138 let op = op.op_str();
2139 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2140 let rn = show_ireg_sized(rn, mb_rru, size);
2141 format!("{} {}, {}", op, rd, rn)
2142 }
2143 &Inst::ULoad8 {
2144 rd,
2145 ref mem,
2146 srcloc: _srcloc,
2147 }
2148 | &Inst::SLoad8 {
2149 rd,
2150 ref mem,
2151 srcloc: _srcloc,
2152 }
2153 | &Inst::ULoad16 {
2154 rd,
2155 ref mem,
2156 srcloc: _srcloc,
2157 }
2158 | &Inst::SLoad16 {
2159 rd,
2160 ref mem,
2161 srcloc: _srcloc,
2162 }
2163 | &Inst::ULoad32 {
2164 rd,
2165 ref mem,
2166 srcloc: _srcloc,
2167 }
2168 | &Inst::SLoad32 {
2169 rd,
2170 ref mem,
2171 srcloc: _srcloc,
2172 }
2173 | &Inst::ULoad64 {
2174 rd,
2175 ref mem,
2176 srcloc: _srcloc,
2177 ..
2178 } => {
2179 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2180
2181 let is_unscaled = match &mem {
2182 &MemArg::Unscaled(..) => true,
2183 _ => false,
2184 };
2185 let (op, size) = match (self, is_unscaled) {
2186 (&Inst::ULoad8 { .. }, false) => ("ldrb", InstSize::Size32),
2187 (&Inst::ULoad8 { .. }, true) => ("ldurb", InstSize::Size32),
2188 (&Inst::SLoad8 { .. }, false) => ("ldrsb", InstSize::Size64),
2189 (&Inst::SLoad8 { .. }, true) => ("ldursb", InstSize::Size64),
2190 (&Inst::ULoad16 { .. }, false) => ("ldrh", InstSize::Size32),
2191 (&Inst::ULoad16 { .. }, true) => ("ldurh", InstSize::Size32),
2192 (&Inst::SLoad16 { .. }, false) => ("ldrsh", InstSize::Size64),
2193 (&Inst::SLoad16 { .. }, true) => ("ldursh", InstSize::Size64),
2194 (&Inst::ULoad32 { .. }, false) => ("ldr", InstSize::Size32),
2195 (&Inst::ULoad32 { .. }, true) => ("ldur", InstSize::Size32),
2196 (&Inst::SLoad32 { .. }, false) => ("ldrsw", InstSize::Size64),
2197 (&Inst::SLoad32 { .. }, true) => ("ldursw", InstSize::Size64),
2198 (&Inst::ULoad64 { .. }, false) => ("ldr", InstSize::Size64),
2199 (&Inst::ULoad64 { .. }, true) => ("ldur", InstSize::Size64),
2200 _ => unreachable!(),
2201 };
2202 let rd = show_ireg_sized(rd.to_reg(), mb_rru, size);
2203 let mem = mem.show_rru(mb_rru);
2204 format!("{}{} {}, {}", mem_str, op, rd, mem)
2205 }
2206 &Inst::Store8 {
2207 rd,
2208 ref mem,
2209 srcloc: _srcloc,
2210 }
2211 | &Inst::Store16 {
2212 rd,
2213 ref mem,
2214 srcloc: _srcloc,
2215 }
2216 | &Inst::Store32 {
2217 rd,
2218 ref mem,
2219 srcloc: _srcloc,
2220 }
2221 | &Inst::Store64 {
2222 rd,
2223 ref mem,
2224 srcloc: _srcloc,
2225 ..
2226 } => {
2227 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2228
2229 let is_unscaled = match &mem {
2230 &MemArg::Unscaled(..) => true,
2231 _ => false,
2232 };
2233 let (op, size) = match (self, is_unscaled) {
2234 (&Inst::Store8 { .. }, false) => ("strb", InstSize::Size32),
2235 (&Inst::Store8 { .. }, true) => ("sturb", InstSize::Size32),
2236 (&Inst::Store16 { .. }, false) => ("strh", InstSize::Size32),
2237 (&Inst::Store16 { .. }, true) => ("sturh", InstSize::Size32),
2238 (&Inst::Store32 { .. }, false) => ("str", InstSize::Size32),
2239 (&Inst::Store32 { .. }, true) => ("stur", InstSize::Size32),
2240 (&Inst::Store64 { .. }, false) => ("str", InstSize::Size64),
2241 (&Inst::Store64 { .. }, true) => ("stur", InstSize::Size64),
2242 _ => unreachable!(),
2243 };
2244 let rd = show_ireg_sized(rd, mb_rru, size);
2245 let mem = mem.show_rru(mb_rru);
2246 format!("{}{} {}, {}", mem_str, op, rd, mem)
2247 }
2248 &Inst::StoreP64 { rt, rt2, ref mem } => {
2249 let rt = rt.show_rru(mb_rru);
2250 let rt2 = rt2.show_rru(mb_rru);
2251 let mem = mem.show_rru_sized(mb_rru, /* size = */ 8);
2252 format!("stp {}, {}, {}", rt, rt2, mem)
2253 }
2254 &Inst::LoadP64 { rt, rt2, ref mem } => {
2255 let rt = rt.to_reg().show_rru(mb_rru);
2256 let rt2 = rt2.to_reg().show_rru(mb_rru);
2257 let mem = mem.show_rru_sized(mb_rru, /* size = */ 8);
2258 format!("ldp {}, {}, {}", rt, rt2, mem)
2259 }
2260 &Inst::Mov { rd, rm } => {
2261 let rd = rd.to_reg().show_rru(mb_rru);
2262 let rm = rm.show_rru(mb_rru);
2263 format!("mov {}, {}", rd, rm)
2264 }
2265 &Inst::Mov32 { rd, rm } => {
2266 let rd = show_ireg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2267 let rm = show_ireg_sized(rm, mb_rru, InstSize::Size32);
2268 format!("mov {}, {}", rd, rm)
2269 }
2270 &Inst::MovZ { rd, ref imm } => {
2271 let rd = rd.to_reg().show_rru(mb_rru);
2272 let imm = imm.show_rru(mb_rru);
2273 format!("movz {}, {}", rd, imm)
2274 }
2275 &Inst::MovN { rd, ref imm } => {
2276 let rd = rd.to_reg().show_rru(mb_rru);
2277 let imm = imm.show_rru(mb_rru);
2278 format!("movn {}, {}", rd, imm)
2279 }
2280 &Inst::MovK { rd, ref imm } => {
2281 let rd = rd.to_reg().show_rru(mb_rru);
2282 let imm = imm.show_rru(mb_rru);
2283 format!("movk {}, {}", rd, imm)
2284 }
2285 &Inst::CSel { rd, rn, rm, cond } => {
2286 let rd = rd.to_reg().show_rru(mb_rru);
2287 let rn = rn.show_rru(mb_rru);
2288 let rm = rm.show_rru(mb_rru);
2289 let cond = cond.show_rru(mb_rru);
2290 format!("csel {}, {}, {}, {}", rd, rn, rm, cond)
2291 }
2292 &Inst::CSet { rd, cond } => {
2293 let rd = rd.to_reg().show_rru(mb_rru);
2294 let cond = cond.show_rru(mb_rru);
2295 format!("cset {}, {}", rd, cond)
2296 }
2297 &Inst::CCmpImm {
2298 size,
2299 rn,
2300 imm,
2301 nzcv,
2302 cond,
2303 } => {
2304 let rn = show_ireg_sized(rn, mb_rru, size);
2305 let imm = imm.show_rru(mb_rru);
2306 let nzcv = nzcv.show_rru(mb_rru);
2307 let cond = cond.show_rru(mb_rru);
2308 format!("ccmp {}, {}, {}, {}", rn, imm, nzcv, cond)
2309 }
2310 &Inst::FpuMove64 { rd, rn } => {
2311 let rd = rd.to_reg().show_rru(mb_rru);
2312 let rn = rn.show_rru(mb_rru);
2313 format!("mov {}.8b, {}.8b", rd, rn)
2314 }
2315 &Inst::FpuMove128 { rd, rn } => {
2316 let rd = rd.to_reg().show_rru(mb_rru);
2317 let rn = rn.show_rru(mb_rru);
2318 format!("mov {}.16b, {}.16b", rd, rn)
2319 }
2320 &Inst::FpuRR { fpu_op, rd, rn } => {
2321 let (op, sizesrc, sizedest) = match fpu_op {
2322 FPUOp1::Abs32 => ("fabs", InstSize::Size32, InstSize::Size32),
2323 FPUOp1::Abs64 => ("fabs", InstSize::Size64, InstSize::Size64),
2324 FPUOp1::Neg32 => ("fneg", InstSize::Size32, InstSize::Size32),
2325 FPUOp1::Neg64 => ("fneg", InstSize::Size64, InstSize::Size64),
2326 FPUOp1::Sqrt32 => ("fsqrt", InstSize::Size32, InstSize::Size32),
2327 FPUOp1::Sqrt64 => ("fsqrt", InstSize::Size64, InstSize::Size64),
2328 FPUOp1::Cvt32To64 => ("fcvt", InstSize::Size32, InstSize::Size64),
2329 FPUOp1::Cvt64To32 => ("fcvt", InstSize::Size64, InstSize::Size32),
2330 };
2331 let rd = show_freg_sized(rd.to_reg(), mb_rru, sizedest);
2332 let rn = show_freg_sized(rn, mb_rru, sizesrc);
2333 format!("{} {}, {}", op, rd, rn)
2334 }
2335 &Inst::FpuRRR { fpu_op, rd, rn, rm } => {
2336 let (op, size) = match fpu_op {
2337 FPUOp2::Add32 => ("fadd", InstSize::Size32),
2338 FPUOp2::Add64 => ("fadd", InstSize::Size64),
2339 FPUOp2::Sub32 => ("fsub", InstSize::Size32),
2340 FPUOp2::Sub64 => ("fsub", InstSize::Size64),
2341 FPUOp2::Mul32 => ("fmul", InstSize::Size32),
2342 FPUOp2::Mul64 => ("fmul", InstSize::Size64),
2343 FPUOp2::Div32 => ("fdiv", InstSize::Size32),
2344 FPUOp2::Div64 => ("fdiv", InstSize::Size64),
2345 FPUOp2::Max32 => ("fmax", InstSize::Size32),
2346 FPUOp2::Max64 => ("fmax", InstSize::Size64),
2347 FPUOp2::Min32 => ("fmin", InstSize::Size32),
2348 FPUOp2::Min64 => ("fmin", InstSize::Size64),
2349 };
2350 let rd = show_freg_sized(rd.to_reg(), mb_rru, size);
2351 let rn = show_freg_sized(rn, mb_rru, size);
2352 let rm = show_freg_sized(rm, mb_rru, size);
2353 format!("{} {}, {}, {}", op, rd, rn, rm)
2354 }
2355 &Inst::FpuRRI { fpu_op, rd, rn } => {
2356 let (op, imm, vector) = match fpu_op {
2357 FPUOpRI::UShr32(imm) => ("ushr", imm.show_rru(mb_rru), true),
2358 FPUOpRI::UShr64(imm) => ("ushr", imm.show_rru(mb_rru), false),
2359 FPUOpRI::Sli32(imm) => ("sli", imm.show_rru(mb_rru), true),
2360 FPUOpRI::Sli64(imm) => ("sli", imm.show_rru(mb_rru), false),
2361 };
2362
2363 let show_vreg_fn: fn(Reg, Option<&RealRegUniverse>) -> String = if vector {
2364 |reg, mb_rru| show_vreg_vector(reg, mb_rru, F32X2)
2365 } else {
2366 show_vreg_scalar
2367 };
2368 let rd = show_vreg_fn(rd.to_reg(), mb_rru);
2369 let rn = show_vreg_fn(rn, mb_rru);
2370 format!("{} {}, {}, {}", op, rd, rn, imm)
2371 }
2372 &Inst::FpuRRRR {
2373 fpu_op,
2374 rd,
2375 rn,
2376 rm,
2377 ra,
2378 } => {
2379 let (op, size) = match fpu_op {
2380 FPUOp3::MAdd32 => ("fmadd", InstSize::Size32),
2381 FPUOp3::MAdd64 => ("fmadd", InstSize::Size64),
2382 };
2383 let rd = show_freg_sized(rd.to_reg(), mb_rru, size);
2384 let rn = show_freg_sized(rn, mb_rru, size);
2385 let rm = show_freg_sized(rm, mb_rru, size);
2386 let ra = show_freg_sized(ra, mb_rru, size);
2387 format!("{} {}, {}, {}, {}", op, rd, rn, rm, ra)
2388 }
2389 &Inst::FpuCmp32 { rn, rm } => {
2390 let rn = show_freg_sized(rn, mb_rru, InstSize::Size32);
2391 let rm = show_freg_sized(rm, mb_rru, InstSize::Size32);
2392 format!("fcmp {}, {}", rn, rm)
2393 }
2394 &Inst::FpuCmp64 { rn, rm } => {
2395 let rn = show_freg_sized(rn, mb_rru, InstSize::Size64);
2396 let rm = show_freg_sized(rm, mb_rru, InstSize::Size64);
2397 format!("fcmp {}, {}", rn, rm)
2398 }
2399 &Inst::FpuLoad32 { rd, ref mem, .. } => {
2400 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2401 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2402 let mem = mem.show_rru(mb_rru);
2403 format!("{}ldr {}, {}", mem_str, rd, mem)
2404 }
2405 &Inst::FpuLoad64 { rd, ref mem, .. } => {
2406 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size64);
2407 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2408 let mem = mem.show_rru(mb_rru);
2409 format!("{}ldr {}, {}", mem_str, rd, mem)
2410 }
2411 &Inst::FpuLoad128 { rd, ref mem, .. } => {
2412 let rd = rd.to_reg().show_rru(mb_rru);
2413 let rd = "q".to_string() + &rd[1..];
2414 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2415 let mem = mem.show_rru(mb_rru);
2416 format!("{}ldr {}, {}", mem_str, rd, mem)
2417 }
2418 &Inst::FpuStore32 { rd, ref mem, .. } => {
2419 let rd = show_freg_sized(rd, mb_rru, InstSize::Size32);
2420 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2421 let mem = mem.show_rru(mb_rru);
2422 format!("{}str {}, {}", mem_str, rd, mem)
2423 }
2424 &Inst::FpuStore64 { rd, ref mem, .. } => {
2425 let rd = show_freg_sized(rd, mb_rru, InstSize::Size64);
2426 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2427 let mem = mem.show_rru(mb_rru);
2428 format!("{}str {}, {}", mem_str, rd, mem)
2429 }
2430 &Inst::FpuStore128 { rd, ref mem, .. } => {
2431 let rd = rd.show_rru(mb_rru);
2432 let rd = "q".to_string() + &rd[1..];
2433 let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru);
2434 let mem = mem.show_rru(mb_rru);
2435 format!("{}str {}, {}", mem_str, rd, mem)
2436 }
2437 &Inst::LoadFpuConst32 { rd, const_data } => {
2438 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2439 format!("ldr {}, pc+8 ; b 8 ; data.f32 {}", rd, const_data)
2440 }
2441 &Inst::LoadFpuConst64 { rd, const_data } => {
2442 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size64);
2443 format!("ldr {}, pc+8 ; b 12 ; data.f64 {}", rd, const_data)
2444 }
2445 &Inst::LoadFpuConst128 { rd, const_data } => {
2446 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size128);
2447 format!("ldr {}, pc+8 ; b 20 ; data.f128 0x{:032x}", rd, const_data)
2448 }
2449 &Inst::FpuToInt { op, rd, rn } => {
2450 let (op, sizesrc, sizedest) = match op {
2451 FpuToIntOp::F32ToI32 => ("fcvtzs", InstSize::Size32, InstSize::Size32),
2452 FpuToIntOp::F32ToU32 => ("fcvtzu", InstSize::Size32, InstSize::Size32),
2453 FpuToIntOp::F32ToI64 => ("fcvtzs", InstSize::Size32, InstSize::Size64),
2454 FpuToIntOp::F32ToU64 => ("fcvtzu", InstSize::Size32, InstSize::Size64),
2455 FpuToIntOp::F64ToI32 => ("fcvtzs", InstSize::Size64, InstSize::Size32),
2456 FpuToIntOp::F64ToU32 => ("fcvtzu", InstSize::Size64, InstSize::Size32),
2457 FpuToIntOp::F64ToI64 => ("fcvtzs", InstSize::Size64, InstSize::Size64),
2458 FpuToIntOp::F64ToU64 => ("fcvtzu", InstSize::Size64, InstSize::Size64),
2459 };
2460 let rd = show_ireg_sized(rd.to_reg(), mb_rru, sizedest);
2461 let rn = show_freg_sized(rn, mb_rru, sizesrc);
2462 format!("{} {}, {}", op, rd, rn)
2463 }
2464 &Inst::IntToFpu { op, rd, rn } => {
2465 let (op, sizesrc, sizedest) = match op {
2466 IntToFpuOp::I32ToF32 => ("scvtf", InstSize::Size32, InstSize::Size32),
2467 IntToFpuOp::U32ToF32 => ("ucvtf", InstSize::Size32, InstSize::Size32),
2468 IntToFpuOp::I64ToF32 => ("scvtf", InstSize::Size64, InstSize::Size32),
2469 IntToFpuOp::U64ToF32 => ("ucvtf", InstSize::Size64, InstSize::Size32),
2470 IntToFpuOp::I32ToF64 => ("scvtf", InstSize::Size32, InstSize::Size64),
2471 IntToFpuOp::U32ToF64 => ("ucvtf", InstSize::Size32, InstSize::Size64),
2472 IntToFpuOp::I64ToF64 => ("scvtf", InstSize::Size64, InstSize::Size64),
2473 IntToFpuOp::U64ToF64 => ("ucvtf", InstSize::Size64, InstSize::Size64),
2474 };
2475 let rd = show_freg_sized(rd.to_reg(), mb_rru, sizedest);
2476 let rn = show_ireg_sized(rn, mb_rru, sizesrc);
2477 format!("{} {}, {}", op, rd, rn)
2478 }
2479 &Inst::FpuCSel32 { rd, rn, rm, cond } => {
2480 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2481 let rn = show_freg_sized(rn, mb_rru, InstSize::Size32);
2482 let rm = show_freg_sized(rm, mb_rru, InstSize::Size32);
2483 let cond = cond.show_rru(mb_rru);
2484 format!("fcsel {}, {}, {}, {}", rd, rn, rm, cond)
2485 }
2486 &Inst::FpuCSel64 { rd, rn, rm, cond } => {
2487 let rd = show_freg_sized(rd.to_reg(), mb_rru, InstSize::Size64);
2488 let rn = show_freg_sized(rn, mb_rru, InstSize::Size64);
2489 let rm = show_freg_sized(rm, mb_rru, InstSize::Size64);
2490 let cond = cond.show_rru(mb_rru);
2491 format!("fcsel {}, {}, {}, {}", rd, rn, rm, cond)
2492 }
2493 &Inst::FpuRound { op, rd, rn } => {
2494 let (inst, size) = match op {
2495 FpuRoundMode::Minus32 => ("frintm", InstSize::Size32),
2496 FpuRoundMode::Minus64 => ("frintm", InstSize::Size64),
2497 FpuRoundMode::Plus32 => ("frintp", InstSize::Size32),
2498 FpuRoundMode::Plus64 => ("frintp", InstSize::Size64),
2499 FpuRoundMode::Zero32 => ("frintz", InstSize::Size32),
2500 FpuRoundMode::Zero64 => ("frintz", InstSize::Size64),
2501 FpuRoundMode::Nearest32 => ("frintn", InstSize::Size32),
2502 FpuRoundMode::Nearest64 => ("frintn", InstSize::Size64),
2503 };
2504 let rd = show_freg_sized(rd.to_reg(), mb_rru, size);
2505 let rn = show_freg_sized(rn, mb_rru, size);
2506 format!("{} {}, {}", inst, rd, rn)
2507 }
2508 &Inst::MovToVec64 { rd, rn } => {
2509 let rd = rd.to_reg().show_rru(mb_rru);
2510 let rn = rn.show_rru(mb_rru);
2511 format!("mov {}.d[0], {}", rd, rn)
2512 }
2513 &Inst::MovFromVec64 { rd, rn } => {
2514 let rd = rd.to_reg().show_rru(mb_rru);
2515 let rn = rn.show_rru(mb_rru);
2516 format!("mov {}, {}.d[0]", rd, rn)
2517 }
2518 &Inst::VecRRR {
2519 rd,
2520 rn,
2521 rm,
2522 alu_op,
2523 ty,
2524 } => {
2525 let (op, vector) = match alu_op {
2526 VecALUOp::SQAddScalar => ("sqadd", false),
2527 VecALUOp::UQAddScalar => ("uqadd", false),
2528 VecALUOp::SQSubScalar => ("sqsub", false),
2529 VecALUOp::UQSubScalar => ("uqsub", false),
2530 VecALUOp::Cmeq => ("cmeq", true),
2531 VecALUOp::Cmge => ("cmge", true),
2532 VecALUOp::Cmgt => ("cmgt", true),
2533 VecALUOp::Cmhs => ("cmhs", true),
2534 VecALUOp::Cmhi => ("cmhi", true),
2535 };
2536
2537 let show_vreg_fn: fn(Reg, Option<&RealRegUniverse>, Type) -> String = if vector {
2538 |reg, mb_rru, ty| show_vreg_vector(reg, mb_rru, ty)
2539 } else {
2540 |reg, mb_rru, _ty| show_vreg_scalar(reg, mb_rru)
2541 };
2542
2543 let rd = show_vreg_fn(rd.to_reg(), mb_rru, ty);
2544 let rn = show_vreg_fn(rn, mb_rru, ty);
2545 let rm = show_vreg_fn(rm, mb_rru, ty);
2546 format!("{} {}, {}, {}", op, rd, rn, rm)
2547 }
2548 &Inst::VecMisc { op, rd, rn, ty } => {
2549 let op = match op {
2550 VecMisc2::Not => "mvn",
2551 };
2552
2553 let rd = show_vreg_vector(rd.to_reg(), mb_rru, ty);
2554 let rn = show_vreg_vector(rn, mb_rru, ty);
2555 format!("{} {}, {}", op, rd, rn)
2556 }
2557 &Inst::MovToNZCV { rn } => {
2558 let rn = rn.show_rru(mb_rru);
2559 format!("msr nzcv, {}", rn)
2560 }
2561 &Inst::MovFromNZCV { rd } => {
2562 let rd = rd.to_reg().show_rru(mb_rru);
2563 format!("mrs {}, nzcv", rd)
2564 }
2565 &Inst::CondSet { rd, cond } => {
2566 let rd = rd.to_reg().show_rru(mb_rru);
2567 let cond = cond.show_rru(mb_rru);
2568 format!("cset {}, {}", rd, cond)
2569 }
2570 &Inst::Extend {
2571 rd,
2572 rn,
2573 signed,
2574 from_bits,
2575 to_bits,
2576 } if from_bits >= 8 => {
2577 // Is the destination a 32-bit register? Corresponds to whether
2578 // extend-to width is <= 32 bits, *unless* we have an unsigned
2579 // 32-to-64-bit extension, which is implemented with a "mov" to a
2580 // 32-bit (W-reg) dest, because this zeroes the top 32 bits.
2581 let dest_size = if !signed && from_bits == 32 && to_bits == 64 {
2582 InstSize::Size32
2583 } else {
2584 InstSize::from_bits(to_bits)
2585 };
2586 let rd = show_ireg_sized(rd.to_reg(), mb_rru, dest_size);
2587 let rn = show_ireg_sized(rn, mb_rru, InstSize::from_bits(from_bits));
2588 let op = match (signed, from_bits, to_bits) {
2589 (false, 8, 32) => "uxtb",
2590 (true, 8, 32) => "sxtb",
2591 (false, 16, 32) => "uxth",
2592 (true, 16, 32) => "sxth",
2593 (false, 8, 64) => "uxtb",
2594 (true, 8, 64) => "sxtb",
2595 (false, 16, 64) => "uxth",
2596 (true, 16, 64) => "sxth",
2597 (false, 32, 64) => "mov", // special case (see above).
2598 (true, 32, 64) => "sxtw",
2599 _ => panic!("Unsupported Extend case: {:?}", self),
2600 };
2601 format!("{} {}, {}", op, rd, rn)
2602 }
2603 &Inst::Extend {
2604 rd,
2605 rn,
2606 signed,
2607 from_bits,
2608 to_bits,
2609 } if from_bits == 1 && signed => {
2610 let dest_size = InstSize::from_bits(to_bits);
2611 let zr = if dest_size.is32() { "wzr" } else { "xzr" };
2612 let rd32 = show_ireg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2613 let rd = show_ireg_sized(rd.to_reg(), mb_rru, dest_size);
2614 let rn = show_ireg_sized(rn, mb_rru, InstSize::Size32);
2615 format!("and {}, {}, #1 ; sub {}, {}, {}", rd32, rn, rd, zr, rd)
2616 }
2617 &Inst::Extend {
2618 rd,
2619 rn,
2620 signed,
2621 from_bits,
2622 ..
2623 } if from_bits == 1 && !signed => {
2624 let rd = show_ireg_sized(rd.to_reg(), mb_rru, InstSize::Size32);
2625 let rn = show_ireg_sized(rn, mb_rru, InstSize::Size32);
2626 format!("and {}, {}, #1", rd, rn)
2627 }
2628 &Inst::Extend { .. } => {
2629 panic!("Unsupported Extend case");
2630 }
2631 &Inst::Call { .. } => format!("bl 0"),
2632 &Inst::CallInd { ref info, .. } => {
2633 let rn = info.rn.show_rru(mb_rru);
2634 format!("blr {}", rn)
2635 }
2636 &Inst::Ret => "ret".to_string(),
2637 &Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(),
2638 &Inst::Jump { ref dest } => {
2639 let dest = dest.show_rru(mb_rru);
2640 format!("b {}", dest)
2641 }
2642 &Inst::CondBr {
2643 ref taken,
2644 ref not_taken,
2645 ref kind,
2646 } => {
2647 let taken = taken.show_rru(mb_rru);
2648 let not_taken = not_taken.show_rru(mb_rru);
2649 match kind {
2650 &CondBrKind::Zero(reg) => {
2651 let reg = reg.show_rru(mb_rru);
2652 format!("cbz {}, {} ; b {}", reg, taken, not_taken)
2653 }
2654 &CondBrKind::NotZero(reg) => {
2655 let reg = reg.show_rru(mb_rru);
2656 format!("cbnz {}, {} ; b {}", reg, taken, not_taken)
2657 }
2658 &CondBrKind::Cond(c) => {
2659 let c = c.show_rru(mb_rru);
2660 format!("b.{} {} ; b {}", c, taken, not_taken)
2661 }
2662 }
2663 }
2664 &Inst::OneWayCondBr {
2665 ref target,
2666 ref kind,
2667 } => {
2668 let target = target.show_rru(mb_rru);
2669 match kind {
2670 &CondBrKind::Zero(reg) => {
2671 let reg = reg.show_rru(mb_rru);
2672 format!("cbz {}, {}", reg, target)
2673 }
2674 &CondBrKind::NotZero(reg) => {
2675 let reg = reg.show_rru(mb_rru);
2676 format!("cbnz {}, {}", reg, target)
2677 }
2678 &CondBrKind::Cond(c) => {
2679 let c = c.show_rru(mb_rru);
2680 format!("b.{} {}", c, target)
2681 }
2682 }
2683 }
2684 &Inst::IndirectBr { rn, .. } => {
2685 let rn = rn.show_rru(mb_rru);
2686 format!("br {}", rn)
2687 }
2688 &Inst::Brk => "brk #0".to_string(),
2689 &Inst::Udf { .. } => "udf".to_string(),
2690 &Inst::Adr { rd, off } => {
2691 let rd = rd.show_rru(mb_rru);
2692 format!("adr {}, pc+{}", rd, off)
2693 }
2694 &Inst::Word4 { data } => format!("data.i32 {}", data),
2695 &Inst::Word8 { data } => format!("data.i64 {}", data),
2696 &Inst::JTSequence {
2697 ref info,
2698 ridx,
2699 rtmp1,
2700 rtmp2,
2701 ..
2702 } => {
2703 let ridx = ridx.show_rru(mb_rru);
2704 let rtmp1 = rtmp1.show_rru(mb_rru);
2705 let rtmp2 = rtmp2.show_rru(mb_rru);
2706 format!(
2707 concat!(
2708 "adr {}, pc+16 ; ",
2709 "ldrsw {}, [{}, {}, LSL 2] ; ",
2710 "add {}, {}, {} ; ",
2711 "br {} ; ",
2712 "jt_entries {:?}"
2713 ),
2714 rtmp1, rtmp2, rtmp1, ridx, rtmp1, rtmp1, rtmp2, rtmp1, info.targets
2715 )
2716 }
2717 &Inst::LoadConst64 { rd, const_data } => {
2718 let rd = rd.show_rru(mb_rru);
2719 format!("ldr {}, 8 ; b 12 ; data {:?}", rd, const_data)
2720 }
2721 &Inst::LoadExtName {
2722 rd,
2723 ref name,
2724 offset,
2725 srcloc: _srcloc,
2726 } => {
2727 let rd = rd.show_rru(mb_rru);
2728 format!("ldr {}, 8 ; b 12 ; data {:?} + {}", rd, name, offset)
2729 }
2730 &Inst::LoadAddr { rd, ref mem } => {
2731 // TODO: we really should find a better way to avoid duplication of
2732 // this logic between `emit()` and `show_rru()` -- a separate 1-to-N
2733 // expansion stage (i.e., legalization, but without the slow edit-in-place
2734 // of the existing legalization framework).
2735 let (mem_insts, mem) = mem_finalize(0, mem, &EmitState::default());
2736 let mut ret = String::new();
2737 for inst in mem_insts.into_iter() {
2738 ret.push_str(&inst.show_rru(mb_rru));
2739 }
2740 let (reg, offset) = match mem {
2741 MemArg::Unscaled(r, simm9) => (r, simm9.value()),
2742 MemArg::UnsignedOffset(r, uimm12scaled) => (r, uimm12scaled.value() as i32),
2743 _ => panic!("Unsupported case for LoadAddr: {:?}", mem),
2744 };
2745 let abs_offset = if offset < 0 {
2746 -offset as u64
2747 } else {
2748 offset as u64
2749 };
2750 let alu_op = if offset < 0 {
2751 ALUOp::Sub64
2752 } else {
2753 ALUOp::Add64
2754 };
2755
2756 if offset == 0 {
2757 let mov = Inst::mov(rd, reg);
2758 ret.push_str(&mov.show_rru(mb_rru));
2759 } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
2760 let add = Inst::AluRRImm12 {
2761 alu_op,
2762 rd,
2763 rn: reg,
2764 imm12,
2765 };
2766 ret.push_str(&add.show_rru(mb_rru));
2767 } else {
2768 let tmp = writable_spilltmp_reg();
2769 for inst in Inst::load_constant(tmp, abs_offset).into_iter() {
2770 ret.push_str(&inst.show_rru(mb_rru));
2771 }
2772 let add = Inst::AluRRR {
2773 alu_op,
2774 rd,
2775 rn: reg,
2776 rm: tmp.to_reg(),
2777 };
2778 ret.push_str(&add.show_rru(mb_rru));
2779 }
2780 ret
2781 }
2782 &Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset),
2783 &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space),
2784 }
2785 }
2786 }
2787
2788 //=============================================================================
2789 // Label fixups and jump veneers.
2790
2791 /// Different forms of label references for different instruction formats.
2792 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2793 pub enum LabelUse {
2794 /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
2795 /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
2796 Branch19,
2797 /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
2798 /// signed bits, in bits 25:0. Used by b, bl.
2799 Branch26,
2800 /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
2801 /// in bits 23:5.
2802 Ldr19,
2803 /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
2804 /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
2805 Adr21,
2806 /// 32-bit PC relative constant offset (from address of constant itself),
2807 /// signed. Used in jump tables.
2808 PCRel32,
2809 }
2810
2811 impl MachInstLabelUse for LabelUse {
2812 /// Alignment for veneer code. Every AArch64 instruction must be 4-byte-aligned.
2813 const ALIGN: CodeOffset = 4;
2814
2815 /// Maximum PC-relative range (positive), inclusive.
max_pos_range(self) -> CodeOffset2816 fn max_pos_range(self) -> CodeOffset {
2817 match self {
2818 // 19-bit immediate, left-shifted by 2, for 21 bits of total range. Signed, so +2^20
2819 // from zero. Likewise for two other shifted cases below.
2820 LabelUse::Branch19 => (1 << 20) - 1,
2821 LabelUse::Branch26 => (1 << 27) - 1,
2822 LabelUse::Ldr19 => (1 << 20) - 1,
2823 // Adr does not shift its immediate, so the 21-bit immediate gives 21 bits of total
2824 // range.
2825 LabelUse::Adr21 => (1 << 20) - 1,
2826 LabelUse::PCRel32 => 0x7fffffff,
2827 }
2828 }
2829
2830 /// Maximum PC-relative range (negative).
max_neg_range(self) -> CodeOffset2831 fn max_neg_range(self) -> CodeOffset {
2832 // All forms are twos-complement signed offsets, so negative limit is one more than
2833 // positive limit.
2834 self.max_pos_range() + 1
2835 }
2836
2837 /// Size of window into code needed to do the patch.
patch_size(self) -> CodeOffset2838 fn patch_size(self) -> CodeOffset {
2839 // Patch is on one instruction only for all of these label reference types.
2840 4
2841 }
2842
2843 /// Perform the patch.
patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset)2844 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2845 let pc_rel = (label_offset as i64) - (use_offset as i64);
2846 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2847 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2848 let pc_rel = pc_rel as u32;
2849 let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2850 let mask = match self {
2851 LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
2852 LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
2853 LabelUse::Ldr19 => 0x00ffffe0, // bits 23..5 inclusive
2854 LabelUse::Adr21 => 0x60ffffe0, // bits 30..29, 25..5 inclusive
2855 LabelUse::PCRel32 => 0xffffffff,
2856 };
2857 let pc_rel_shifted = match self {
2858 LabelUse::Adr21 | LabelUse::PCRel32 => pc_rel,
2859 _ => {
2860 debug_assert!(pc_rel & 3 == 0);
2861 pc_rel >> 2
2862 }
2863 };
2864 let pc_rel_inserted = match self {
2865 LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
2866 LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
2867 LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
2868 LabelUse::PCRel32 => pc_rel_shifted,
2869 };
2870 let is_add = match self {
2871 LabelUse::PCRel32 => true,
2872 _ => false,
2873 };
2874 let insn_word = if is_add {
2875 insn_word.wrapping_add(pc_rel_inserted)
2876 } else {
2877 (insn_word & !mask) | pc_rel_inserted
2878 };
2879 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
2880 }
2881
2882 /// Is a veneer supported for this label reference type?
supports_veneer(self) -> bool2883 fn supports_veneer(self) -> bool {
2884 match self {
2885 LabelUse::Branch19 => true, // veneer is a Branch26
2886 _ => false,
2887 }
2888 }
2889
2890 /// How large is the veneer, if supported?
veneer_size(self) -> CodeOffset2891 fn veneer_size(self) -> CodeOffset {
2892 4
2893 }
2894
2895 /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
2896 /// an offset and label-use for the veneer's use of the original label.
generate_veneer( self, buffer: &mut [u8], veneer_offset: CodeOffset, ) -> (CodeOffset, LabelUse)2897 fn generate_veneer(
2898 self,
2899 buffer: &mut [u8],
2900 veneer_offset: CodeOffset,
2901 ) -> (CodeOffset, LabelUse) {
2902 match self {
2903 LabelUse::Branch19 => {
2904 // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
2905 // bother with constructing an Inst.
2906 let insn_word = 0b000101 << 26;
2907 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
2908 (veneer_offset, LabelUse::Branch26)
2909 }
2910 _ => panic!("Unsupported label-reference type for veneer generation!"),
2911 }
2912 }
2913 }
2914