1 //! This module defines x86_64-specific machine instruction types.
2
3 use crate::binemit::{CodeOffset, StackMap};
4 use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
5 use crate::isa::unwind::UnwindInst;
6 use crate::isa::x64::abi::X64ABIMachineSpec;
7 use crate::isa::x64::settings as x64_settings;
8 use crate::isa::CallConv;
9 use crate::machinst::*;
10 use crate::{settings, settings::Flags, CodegenError, CodegenResult};
11 use alloc::boxed::Box;
12 use alloc::vec::Vec;
13 use regalloc::{
14 PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector,
15 RegUsageMapper, SpillSlot, VirtualReg, Writable,
16 };
17 use smallvec::{smallvec, SmallVec};
18 use std::fmt;
19 use std::string::{String, ToString};
20
21 pub mod args;
22 mod emit;
23 #[cfg(test)]
24 mod emit_tests;
25 pub mod regs;
26 pub mod unwind;
27
28 use args::*;
29 use regs::{create_reg_universe_systemv, show_ireg_sized};
30
31 //=============================================================================
32 // Instructions (top level): definition
33
34 // Don't build these directly. Instead use the Inst:: functions to create them.
35
36 /// Instructions. Destinations are on the RIGHT (a la AT&T syntax).
37 #[derive(Clone)]
38 pub enum Inst {
39 /// Nops of various sizes, including zero.
40 Nop { len: u8 },
41
42 // =====================================
43 // Integer instructions.
44 /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg
45 AluRmiR {
46 size: OperandSize, // 4 or 8
47 op: AluRmiROpcode,
48 src: RegMemImm,
49 dst: Writable<Reg>,
50 },
51
52 /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc.
53 UnaryRmR {
54 size: OperandSize, // 2, 4 or 8
55 op: UnaryRmROpcode,
56 src: RegMem,
57 dst: Writable<Reg>,
58 },
59
60 /// Bitwise not
61 Not {
62 size: OperandSize, // 1, 2, 4 or 8
63 src: Writable<Reg>,
64 },
65
66 /// Integer negation
67 Neg {
68 size: OperandSize, // 1, 2, 4 or 8
69 src: Writable<Reg>,
70 },
71
72 /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
73 Div {
74 size: OperandSize, // 1, 2, 4 or 8
75 signed: bool,
76 divisor: RegMem,
77 },
78
79 /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
80 MulHi {
81 size: OperandSize, // 2, 4, or 8
82 signed: bool,
83 rhs: RegMem,
84 },
85
86 /// A synthetic sequence to implement the right inline checks for remainder and division,
87 /// assuming the dividend is in %rax.
88 /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div
89 /// instruction does.
90 /// The generated code sequence is described in the emit's function match arm for this
91 /// instruction.
92 ///
93 /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem
94 /// with the temporary and divisor registers. Make sure to zero %rdx right before this
95 /// instruction, or you might run into regalloc failures where %rdx is live before its first
96 /// def!
97 CheckedDivOrRemSeq {
98 kind: DivOrRemKind,
99 size: OperandSize,
100 /// The divisor operand. Note it's marked as modified so that it gets assigned a register
101 /// different from the temporary.
102 divisor: Writable<Reg>,
103 tmp: Option<Writable<Reg>>,
104 },
105
106 /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo)
107 /// or al into ah: (cbw)
108 SignExtendData {
109 size: OperandSize, // 1, 2, 4 or 8
110 },
111
112 /// Constant materialization: (imm32 imm64) reg.
113 /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
114 Imm {
115 dst_size: OperandSize, // 4 or 8
116 simm64: u64,
117 dst: Writable<Reg>,
118 },
119
120 /// GPR to GPR move: mov (64 32) reg reg.
121 MovRR {
122 size: OperandSize, // 4 or 8
123 src: Reg,
124 dst: Writable<Reg>,
125 },
126
127 /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg.
128 /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it
129 /// unnecessary. For that case we emit the equivalent "movl AM, reg32".
130 MovzxRmR {
131 ext_mode: ExtMode,
132 src: RegMem,
133 dst: Writable<Reg>,
134 },
135
136 /// A plain 64-bit integer load, since MovZX_RM_R can't represent that.
137 Mov64MR {
138 src: SyntheticAmode,
139 dst: Writable<Reg>,
140 },
141
142 /// Loads the memory address of addr into dst.
143 LoadEffectiveAddress {
144 addr: SyntheticAmode,
145 dst: Writable<Reg>,
146 },
147
148 /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
149 MovsxRmR {
150 ext_mode: ExtMode,
151 src: RegMem,
152 dst: Writable<Reg>,
153 },
154
155 /// Integer stores: mov (b w l q) reg addr.
156 MovRM {
157 size: OperandSize, // 1, 2, 4 or 8.
158 src: Reg,
159 dst: SyntheticAmode,
160 },
161
162 /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
163 ShiftR {
164 size: OperandSize, // 1, 2, 4 or 8
165 kind: ShiftKind,
166 /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl".
167 num_bits: Option<u8>,
168 dst: Writable<Reg>,
169 },
170
171 /// Arithmetic SIMD shifts.
172 XmmRmiReg {
173 opcode: SseOpcode,
174 src: RegMemImm,
175 dst: Writable<Reg>,
176 },
177
178 /// Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
179 CmpRmiR {
180 size: OperandSize, // 1, 2, 4 or 8
181 opcode: CmpOpcode,
182 src: RegMemImm,
183 dst: Reg,
184 },
185
186 /// Materializes the requested condition code in the destination reg.
187 Setcc { cc: CC, dst: Writable<Reg> },
188
189 /// Integer conditional move.
190 /// Overwrites the destination register.
191 Cmove {
192 size: OperandSize, // 2, 4, or 8
193 cc: CC,
194 src: RegMem,
195 dst: Writable<Reg>,
196 },
197
198 // =====================================
199 // Stack manipulation.
200 /// pushq (reg addr imm)
201 Push64 { src: RegMemImm },
202
203 /// popq reg
204 Pop64 { dst: Writable<Reg> },
205
206 // =====================================
207 // Floating-point operations.
208 /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg
209 XmmRmR {
210 op: SseOpcode,
211 src: RegMem,
212 dst: Writable<Reg>,
213 },
214
215 XmmRmREvex {
216 op: Avx512Opcode,
217 src1: RegMem,
218 src2: Reg,
219 dst: Writable<Reg>,
220 },
221
222 /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt,
223 /// etc.
224 ///
225 /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the
226 /// computation of the instruction dst value and so does not have to be a previously valid
227 /// value. This is characteristic of mov instructions.
228 XmmUnaryRmR {
229 op: SseOpcode,
230 src: RegMem,
231 dst: Writable<Reg>,
232 },
233
234 XmmUnaryRmREvex {
235 op: Avx512Opcode,
236 src: RegMem,
237 dst: Writable<Reg>,
238 },
239
240 /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq
241 XmmMovRM {
242 op: SseOpcode,
243 src: Reg,
244 dst: SyntheticAmode,
245 },
246
247 /// XMM (vector) unary op (to move a constant value into an xmm register): movups
248 XmmLoadConst {
249 src: VCodeConstant,
250 dst: Writable<Reg>,
251 ty: Type,
252 },
253
254 /// XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si
255 XmmToGpr {
256 op: SseOpcode,
257 src: Reg,
258 dst: Writable<Reg>,
259 dst_size: OperandSize,
260 },
261
262 /// XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d}
263 GprToXmm {
264 op: SseOpcode,
265 src: RegMem,
266 dst: Writable<Reg>,
267 src_size: OperandSize,
268 },
269
270 /// Converts an unsigned int64 to a float32/float64.
271 CvtUint64ToFloatSeq {
272 dst_size: OperandSize, // 4 or 8
273 /// A copy of the source register, fed by lowering. It is marked as modified during
274 /// register allocation to make sure that the temporary registers differ from the src
275 /// register, since both registers are live at the same time in the generated code
276 /// sequence.
277 src: Writable<Reg>,
278 dst: Writable<Reg>,
279 tmp_gpr1: Writable<Reg>,
280 tmp_gpr2: Writable<Reg>,
281 },
282
283 /// Converts a scalar xmm to a signed int32/int64.
284 CvtFloatToSintSeq {
285 dst_size: OperandSize,
286 src_size: OperandSize,
287 is_saturating: bool,
288 /// A copy of the source register, fed by lowering. It is marked as modified during
289 /// register allocation to make sure that the temporary xmm register differs from the src
290 /// register, since both registers are live at the same time in the generated code
291 /// sequence.
292 src: Writable<Reg>,
293 dst: Writable<Reg>,
294 tmp_gpr: Writable<Reg>,
295 tmp_xmm: Writable<Reg>,
296 },
297
298 /// Converts a scalar xmm to an unsigned int32/int64.
299 CvtFloatToUintSeq {
300 src_size: OperandSize,
301 dst_size: OperandSize,
302 is_saturating: bool,
303 /// A copy of the source register, fed by lowering, reused as a temporary. It is marked as
304 /// modified during register allocation to make sure that the temporary xmm register
305 /// differs from the src register, since both registers are live at the same time in the
306 /// generated code sequence.
307 src: Writable<Reg>,
308 dst: Writable<Reg>,
309 tmp_gpr: Writable<Reg>,
310 tmp_xmm: Writable<Reg>,
311 },
312
313 /// A sequence to compute min/max with the proper NaN semantics for xmm registers.
314 XmmMinMaxSeq {
315 size: OperandSize,
316 is_min: bool,
317 lhs: Reg,
318 rhs_dst: Writable<Reg>,
319 },
320
321 /// XMM (scalar) conditional move.
322 /// Overwrites the destination register if cc is set.
323 XmmCmove {
324 size: OperandSize, // 4 or 8
325 cc: CC,
326 src: RegMem,
327 dst: Writable<Reg>,
328 },
329
330 /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
331 XmmCmpRmR {
332 op: SseOpcode,
333 src: RegMem,
334 dst: Reg,
335 },
336
337 /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg
338 XmmRmRImm {
339 op: SseOpcode,
340 src: RegMem,
341 dst: Writable<Reg>,
342 imm: u8,
343 size: OperandSize, // 4 or 8
344 },
345
346 // =====================================
347 // Control flow instructions.
348 /// Direct call: call simm32.
349 CallKnown {
350 dest: ExternalName,
351 uses: Vec<Reg>,
352 defs: Vec<Writable<Reg>>,
353 opcode: Opcode,
354 },
355
356 /// Indirect call: callq (reg mem).
357 CallUnknown {
358 dest: RegMem,
359 uses: Vec<Reg>,
360 defs: Vec<Writable<Reg>>,
361 opcode: Opcode,
362 },
363
364 /// Return.
365 Ret,
366
367 /// A placeholder instruction, generating no code, meaning that a function epilogue must be
368 /// inserted there.
369 EpiloguePlaceholder,
370
371 /// Jump to a known target: jmp simm32.
372 JmpKnown { dst: MachLabel },
373
374 /// One-way conditional branch: jcond cond target.
375 ///
376 /// This instruction is useful when we have conditional jumps depending on more than two
377 /// conditions, see for instance the lowering of Brz/brnz with Fcmp inputs.
378 ///
379 /// A note of caution: in contexts where the branch target is another block, this has to be the
380 /// same successor as the one specified in the terminator branch of the current block.
381 /// Otherwise, this might confuse register allocation by creating new invisible edges.
382 JmpIf { cc: CC, taken: MachLabel },
383
384 /// Two-way conditional branch: jcond cond target target.
385 /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate.
386 JmpCond {
387 cc: CC,
388 taken: MachLabel,
389 not_taken: MachLabel,
390 },
391
392 /// Jump-table sequence, as one compound instruction (see note in lower.rs for rationale).
393 /// The generated code sequence is described in the emit's function match arm for this
394 /// instruction.
395 /// See comment in lowering about the temporaries signedness.
396 JmpTableSeq {
397 idx: Reg,
398 tmp1: Writable<Reg>,
399 tmp2: Writable<Reg>,
400 default_target: MachLabel,
401 targets: Vec<MachLabel>,
402 targets_for_term: Vec<MachLabel>,
403 },
404
405 /// Indirect jump: jmpq (reg mem).
406 JmpUnknown { target: RegMem },
407
408 /// Traps if the condition code is set.
409 TrapIf { cc: CC, trap_code: TrapCode },
410
411 /// A debug trap.
412 Hlt,
413
414 /// An instruction that will always trigger the illegal instruction exception.
415 Ud2 { trap_code: TrapCode },
416
417 /// Loads an external symbol in a register, with a relocation:
418 ///
419 /// movq $name@GOTPCREL(%rip), dst if PIC is enabled, or
420 /// movabsq $name, dst otherwise.
421 LoadExtName {
422 dst: Writable<Reg>,
423 name: Box<ExternalName>,
424 offset: i64,
425 },
426
427 // =====================================
428 // Instructions pertaining to atomic memory accesses.
429 /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions:
430 ///
431 /// `dst` (read) address
432 /// `src` (read) replacement value
433 /// %rax (modified) in: expected value, out: value that was actually at `dst`
434 /// %rflags is written. Do not assume anything about it after the instruction.
435 ///
436 /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as
437 /// they were before.
438 LockCmpxchg {
439 ty: Type, // I8, I16, I32 or I64
440 src: Reg,
441 dst: SyntheticAmode,
442 },
443
444 /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction.
445 /// This atomically modifies a value in memory and returns the old value. The sequence
446 /// consists of an initial "normal" load from `dst`, followed by a loop which computes the
447 /// new value and tries to compare-and-swap ("CAS") it into `dst`, using the native
448 /// instruction `lock cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful.
449 /// If there is no contention, there will be only one pass through the loop body. The
450 /// sequence does *not* perform any explicit memory fence instructions
451 /// (mfence/sfence/lfence).
452 ///
453 /// Note that the transaction is atomic in the sense that, as observed by some other thread,
454 /// `dst` either has the initial or final value, but no other. It isn't atomic in the sense
455 /// of guaranteeing that no other thread writes to `dst` in between the initial load and the
456 /// CAS -- but that would cause the CAS to fail unless the other thread's last write before
457 /// the CAS wrote the same value that was already there. In other words, this
458 /// implementation suffers (unavoidably) from the A-B-A problem.
459 ///
460 /// This instruction sequence has fixed register uses as follows:
461 ///
462 /// %r9 (read) address
463 /// %r10 (read) second operand for `op`
464 /// %r11 (written) scratch reg; value afterwards has no meaning
465 /// %rax (written) the old value at %r9
466 /// %rflags is written. Do not assume anything about it after the instruction.
467 AtomicRmwSeq {
468 ty: Type, // I8, I16, I32 or I64
469 op: inst_common::AtomicRmwOp,
470 },
471
472 /// A memory fence (mfence, lfence or sfence).
473 Fence { kind: FenceKind },
474
475 // =====================================
476 // Meta-instructions generating no code.
477 /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This
478 /// controls how MemArg::NominalSPOffset args are lowered.
479 VirtualSPOffsetAdj { offset: i64 },
480
481 /// Provides a way to tell the register allocator that the upcoming sequence of instructions
482 /// will overwrite `dst` so it should be considered as a `def`; use this with care.
483 ///
484 /// This is useful when we have a sequence of instructions whose register usages are nominally
485 /// `mod`s, but such that the combination of operations creates a result that is independent of
486 /// the initial register value. It's thus semantically a `def`, not a `mod`, when all the
487 /// instructions are taken together, so we want to ensure the register is defined (its
488 /// live-range starts) prior to the sequence to keep analyses happy.
489 ///
490 /// One alternative would be a compound instruction that somehow encapsulates the others and
491 /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no
492 /// longer flat) and requires knowledge about semantics and initial-value independence anyway.
493 XmmUninitializedValue { dst: Writable<Reg> },
494
495 /// A call to the `ElfTlsGetAddr` libcall. Returns address
496 /// of TLS symbol in rax.
497 ElfTlsGetAddr { symbol: ExternalName },
498
499 /// A Mach-O TLS symbol access. Returns address of the TLS
500 /// symbol in rax.
501 MachOTlsGetAddr { symbol: ExternalName },
502
503 /// A definition of a value label.
504 ValueLabelMarker { reg: Reg, label: ValueLabel },
505
506 /// An unwind pseudoinstruction describing the state of the
507 /// machine at this program point.
508 Unwind { inst: UnwindInst },
509 }
510
low32_will_sign_extend_to_64(x: u64) -> bool511 pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
512 let xs = x as i64;
513 xs == ((xs << 32) >> 32)
514 }
515
516 impl Inst {
517 /// Retrieve a list of ISA feature sets in which the instruction is available. An empty list
518 /// indicates that the instruction is available in the baseline feature set (i.e. SSE2 and
519 /// below); more than one `InstructionSet` in the list indicates that the instruction is present
520 /// *any* of the included ISA feature sets.
available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]>521 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> {
522 match self {
523 // These instructions are part of SSE2, which is a basic requirement in Cranelift, and
524 // don't have to be checked.
525 Inst::AluRmiR { .. }
526 | Inst::AtomicRmwSeq { .. }
527 | Inst::CallKnown { .. }
528 | Inst::CallUnknown { .. }
529 | Inst::CheckedDivOrRemSeq { .. }
530 | Inst::Cmove { .. }
531 | Inst::CmpRmiR { .. }
532 | Inst::CvtFloatToSintSeq { .. }
533 | Inst::CvtFloatToUintSeq { .. }
534 | Inst::CvtUint64ToFloatSeq { .. }
535 | Inst::Div { .. }
536 | Inst::EpiloguePlaceholder
537 | Inst::Fence { .. }
538 | Inst::Hlt
539 | Inst::Imm { .. }
540 | Inst::JmpCond { .. }
541 | Inst::JmpIf { .. }
542 | Inst::JmpKnown { .. }
543 | Inst::JmpTableSeq { .. }
544 | Inst::JmpUnknown { .. }
545 | Inst::LoadEffectiveAddress { .. }
546 | Inst::LoadExtName { .. }
547 | Inst::LockCmpxchg { .. }
548 | Inst::Mov64MR { .. }
549 | Inst::MovRM { .. }
550 | Inst::MovRR { .. }
551 | Inst::MovsxRmR { .. }
552 | Inst::MovzxRmR { .. }
553 | Inst::MulHi { .. }
554 | Inst::Neg { .. }
555 | Inst::Not { .. }
556 | Inst::Nop { .. }
557 | Inst::Pop64 { .. }
558 | Inst::Push64 { .. }
559 | Inst::Ret
560 | Inst::Setcc { .. }
561 | Inst::ShiftR { .. }
562 | Inst::SignExtendData { .. }
563 | Inst::TrapIf { .. }
564 | Inst::Ud2 { .. }
565 | Inst::VirtualSPOffsetAdj { .. }
566 | Inst::XmmCmove { .. }
567 | Inst::XmmCmpRmR { .. }
568 | Inst::XmmLoadConst { .. }
569 | Inst::XmmMinMaxSeq { .. }
570 | Inst::XmmUninitializedValue { .. }
571 | Inst::ElfTlsGetAddr { .. }
572 | Inst::MachOTlsGetAddr { .. }
573 | Inst::ValueLabelMarker { .. }
574 | Inst::Unwind { .. } => smallvec![],
575
576 Inst::UnaryRmR { op, .. } => op.available_from(),
577
578 // These use dynamic SSE opcodes.
579 Inst::GprToXmm { op, .. }
580 | Inst::XmmMovRM { op, .. }
581 | Inst::XmmRmiReg { opcode: op, .. }
582 | Inst::XmmRmR { op, .. }
583 | Inst::XmmRmRImm { op, .. }
584 | Inst::XmmToGpr { op, .. }
585 | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],
586
587 Inst::XmmUnaryRmREvex { op, .. } | Inst::XmmRmREvex { op, .. } => op.available_from(),
588 }
589 }
590 }
591
592 // Handy constructors for Insts.
593
594 impl Inst {
nop(len: u8) -> Self595 pub(crate) fn nop(len: u8) -> Self {
596 debug_assert!(len <= 15);
597 Self::Nop { len }
598 }
599
alu_rmi_r( size: OperandSize, op: AluRmiROpcode, src: RegMemImm, dst: Writable<Reg>, ) -> Self600 pub(crate) fn alu_rmi_r(
601 size: OperandSize,
602 op: AluRmiROpcode,
603 src: RegMemImm,
604 dst: Writable<Reg>,
605 ) -> Self {
606 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
607 src.assert_regclass_is(RegClass::I64);
608 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
609 Self::AluRmiR { size, op, src, dst }
610 }
611
unary_rm_r( size: OperandSize, op: UnaryRmROpcode, src: RegMem, dst: Writable<Reg>, ) -> Self612 pub(crate) fn unary_rm_r(
613 size: OperandSize,
614 op: UnaryRmROpcode,
615 src: RegMem,
616 dst: Writable<Reg>,
617 ) -> Self {
618 src.assert_regclass_is(RegClass::I64);
619 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
620 debug_assert!(size.is_one_of(&[
621 OperandSize::Size16,
622 OperandSize::Size32,
623 OperandSize::Size64
624 ]));
625 Self::UnaryRmR { size, op, src, dst }
626 }
627
not(size: OperandSize, src: Writable<Reg>) -> Inst628 pub(crate) fn not(size: OperandSize, src: Writable<Reg>) -> Inst {
629 debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
630 Inst::Not { size, src }
631 }
632
neg(size: OperandSize, src: Writable<Reg>) -> Inst633 pub(crate) fn neg(size: OperandSize, src: Writable<Reg>) -> Inst {
634 debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
635 Inst::Neg { size, src }
636 }
637
div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst638 pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst {
639 divisor.assert_regclass_is(RegClass::I64);
640 Inst::Div {
641 size,
642 signed,
643 divisor,
644 }
645 }
646
mul_hi(size: OperandSize, signed: bool, rhs: RegMem) -> Inst647 pub(crate) fn mul_hi(size: OperandSize, signed: bool, rhs: RegMem) -> Inst {
648 debug_assert!(size.is_one_of(&[
649 OperandSize::Size16,
650 OperandSize::Size32,
651 OperandSize::Size64
652 ]));
653 rhs.assert_regclass_is(RegClass::I64);
654 Inst::MulHi { size, signed, rhs }
655 }
656
checked_div_or_rem_seq( kind: DivOrRemKind, size: OperandSize, divisor: Writable<Reg>, tmp: Option<Writable<Reg>>, ) -> Inst657 pub(crate) fn checked_div_or_rem_seq(
658 kind: DivOrRemKind,
659 size: OperandSize,
660 divisor: Writable<Reg>,
661 tmp: Option<Writable<Reg>>,
662 ) -> Inst {
663 debug_assert!(divisor.to_reg().get_class() == RegClass::I64);
664 debug_assert!(tmp
665 .map(|tmp| tmp.to_reg().get_class() == RegClass::I64)
666 .unwrap_or(true));
667 Inst::CheckedDivOrRemSeq {
668 kind,
669 size,
670 divisor,
671 tmp,
672 }
673 }
674
sign_extend_data(size: OperandSize) -> Inst675 pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
676 Inst::SignExtendData { size }
677 }
678
imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst679 pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
680 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
681 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
682 // Try to generate a 32-bit immediate when the upper high bits are zeroed (which matches
683 // the semantics of movl).
684 let dst_size = match dst_size {
685 OperandSize::Size64 if simm64 > u32::max_value() as u64 => OperandSize::Size64,
686 _ => OperandSize::Size32,
687 };
688 Inst::Imm {
689 dst_size,
690 simm64,
691 dst,
692 }
693 }
694
mov_r_r(size: OperandSize, src: Reg, dst: Writable<Reg>) -> Inst695 pub(crate) fn mov_r_r(size: OperandSize, src: Reg, dst: Writable<Reg>) -> Inst {
696 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
697 debug_assert!(src.get_class() == RegClass::I64);
698 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
699 Inst::MovRR { size, src, dst }
700 }
701
702 // TODO Can be replaced by `Inst::move` (high-level) and `Inst::unary_rm_r` (low-level)
xmm_mov(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst703 pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
704 src.assert_regclass_is(RegClass::V128);
705 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
706 Inst::XmmUnaryRmR { op, src, dst }
707 }
708
xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst709 pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst {
710 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
711 debug_assert!(ty.is_vector() && ty.bits() == 128);
712 Inst::XmmLoadConst { src, dst, ty }
713 }
714
715 /// Convenient helper for unary float operations.
xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst716 pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
717 src.assert_regclass_is(RegClass::V128);
718 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
719 Inst::XmmUnaryRmR { op, src, dst }
720 }
721
xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable<Reg>) -> Inst722 pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable<Reg>) -> Inst {
723 src.assert_regclass_is(RegClass::V128);
724 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
725 Inst::XmmUnaryRmREvex { op, src, dst }
726 }
727
xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self728 pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
729 src.assert_regclass_is(RegClass::V128);
730 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
731 Inst::XmmRmR { op, src, dst }
732 }
733
xmm_rm_r_evex( op: Avx512Opcode, src1: RegMem, src2: Reg, dst: Writable<Reg>, ) -> Self734 pub(crate) fn xmm_rm_r_evex(
735 op: Avx512Opcode,
736 src1: RegMem,
737 src2: Reg,
738 dst: Writable<Reg>,
739 ) -> Self {
740 src1.assert_regclass_is(RegClass::V128);
741 debug_assert!(src2.get_class() == RegClass::V128);
742 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
743 Inst::XmmRmREvex {
744 op,
745 src1,
746 src2,
747 dst,
748 }
749 }
750
xmm_uninit_value(dst: Writable<Reg>) -> Self751 pub(crate) fn xmm_uninit_value(dst: Writable<Reg>) -> Self {
752 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
753 Inst::XmmUninitializedValue { dst }
754 }
755
xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst756 pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
757 debug_assert!(src.get_class() == RegClass::V128);
758 Inst::XmmMovRM {
759 op,
760 src,
761 dst: dst.into(),
762 }
763 }
764
xmm_to_gpr( op: SseOpcode, src: Reg, dst: Writable<Reg>, dst_size: OperandSize, ) -> Inst765 pub(crate) fn xmm_to_gpr(
766 op: SseOpcode,
767 src: Reg,
768 dst: Writable<Reg>,
769 dst_size: OperandSize,
770 ) -> Inst {
771 debug_assert!(src.get_class() == RegClass::V128);
772 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
773 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
774 Inst::XmmToGpr {
775 op,
776 src,
777 dst,
778 dst_size,
779 }
780 }
781
gpr_to_xmm( op: SseOpcode, src: RegMem, src_size: OperandSize, dst: Writable<Reg>, ) -> Inst782 pub(crate) fn gpr_to_xmm(
783 op: SseOpcode,
784 src: RegMem,
785 src_size: OperandSize,
786 dst: Writable<Reg>,
787 ) -> Inst {
788 src.assert_regclass_is(RegClass::I64);
789 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
790 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
791 Inst::GprToXmm {
792 op,
793 src,
794 dst,
795 src_size,
796 }
797 }
798
xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst799 pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
800 src.assert_regclass_is(RegClass::V128);
801 debug_assert!(dst.get_class() == RegClass::V128);
802 Inst::XmmCmpRmR { op, src, dst }
803 }
804
cvt_u64_to_float_seq( dst_size: OperandSize, src: Writable<Reg>, tmp_gpr1: Writable<Reg>, tmp_gpr2: Writable<Reg>, dst: Writable<Reg>, ) -> Inst805 pub(crate) fn cvt_u64_to_float_seq(
806 dst_size: OperandSize,
807 src: Writable<Reg>,
808 tmp_gpr1: Writable<Reg>,
809 tmp_gpr2: Writable<Reg>,
810 dst: Writable<Reg>,
811 ) -> Inst {
812 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
813 debug_assert!(src.to_reg().get_class() == RegClass::I64);
814 debug_assert!(tmp_gpr1.to_reg().get_class() == RegClass::I64);
815 debug_assert!(tmp_gpr2.to_reg().get_class() == RegClass::I64);
816 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
817 Inst::CvtUint64ToFloatSeq {
818 src,
819 dst,
820 tmp_gpr1,
821 tmp_gpr2,
822 dst_size,
823 }
824 }
825
cvt_float_to_sint_seq( src_size: OperandSize, dst_size: OperandSize, is_saturating: bool, src: Writable<Reg>, dst: Writable<Reg>, tmp_gpr: Writable<Reg>, tmp_xmm: Writable<Reg>, ) -> Inst826 pub(crate) fn cvt_float_to_sint_seq(
827 src_size: OperandSize,
828 dst_size: OperandSize,
829 is_saturating: bool,
830 src: Writable<Reg>,
831 dst: Writable<Reg>,
832 tmp_gpr: Writable<Reg>,
833 tmp_xmm: Writable<Reg>,
834 ) -> Inst {
835 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
836 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
837 debug_assert!(src.to_reg().get_class() == RegClass::V128);
838 debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
839 debug_assert!(tmp_gpr.to_reg().get_class() == RegClass::I64);
840 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
841 Inst::CvtFloatToSintSeq {
842 src_size,
843 dst_size,
844 is_saturating,
845 src,
846 dst,
847 tmp_gpr,
848 tmp_xmm,
849 }
850 }
851
cvt_float_to_uint_seq( src_size: OperandSize, dst_size: OperandSize, is_saturating: bool, src: Writable<Reg>, dst: Writable<Reg>, tmp_gpr: Writable<Reg>, tmp_xmm: Writable<Reg>, ) -> Inst852 pub(crate) fn cvt_float_to_uint_seq(
853 src_size: OperandSize,
854 dst_size: OperandSize,
855 is_saturating: bool,
856 src: Writable<Reg>,
857 dst: Writable<Reg>,
858 tmp_gpr: Writable<Reg>,
859 tmp_xmm: Writable<Reg>,
860 ) -> Inst {
861 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
862 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
863 debug_assert!(src.to_reg().get_class() == RegClass::V128);
864 debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
865 debug_assert!(tmp_gpr.to_reg().get_class() == RegClass::I64);
866 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
867 Inst::CvtFloatToUintSeq {
868 src_size,
869 dst_size,
870 is_saturating,
871 src,
872 dst,
873 tmp_gpr,
874 tmp_xmm,
875 }
876 }
877
xmm_min_max_seq( size: OperandSize, is_min: bool, lhs: Reg, rhs_dst: Writable<Reg>, ) -> Inst878 pub(crate) fn xmm_min_max_seq(
879 size: OperandSize,
880 is_min: bool,
881 lhs: Reg,
882 rhs_dst: Writable<Reg>,
883 ) -> Inst {
884 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
885 debug_assert_eq!(lhs.get_class(), RegClass::V128);
886 debug_assert_eq!(rhs_dst.to_reg().get_class(), RegClass::V128);
887 Inst::XmmMinMaxSeq {
888 size,
889 is_min,
890 lhs,
891 rhs_dst,
892 }
893 }
894
xmm_rm_r_imm( op: SseOpcode, src: RegMem, dst: Writable<Reg>, imm: u8, size: OperandSize, ) -> Inst895 pub(crate) fn xmm_rm_r_imm(
896 op: SseOpcode,
897 src: RegMem,
898 dst: Writable<Reg>,
899 imm: u8,
900 size: OperandSize,
901 ) -> Inst {
902 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
903 Inst::XmmRmRImm {
904 op,
905 src,
906 dst,
907 imm,
908 size,
909 }
910 }
911
movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst912 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
913 src.assert_regclass_is(RegClass::I64);
914 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
915 Inst::MovzxRmR { ext_mode, src, dst }
916 }
917
xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst918 pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst {
919 src.assert_regclass_is(RegClass::V128);
920 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
921 Inst::XmmRmiReg { opcode, src, dst }
922 }
923
movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst924 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
925 src.assert_regclass_is(RegClass::I64);
926 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
927 Inst::MovsxRmR { ext_mode, src, dst }
928 }
929
mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst930 pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
931 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
932 Inst::Mov64MR {
933 src: src.into(),
934 dst,
935 }
936 }
937
938 /// A convenience function to be able to use a RegMem as the source of a move.
mov64_rm_r(src: RegMem, dst: Writable<Reg>) -> Inst939 pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>) -> Inst {
940 src.assert_regclass_is(RegClass::I64);
941 match src {
942 RegMem::Reg { reg } => Self::mov_r_r(OperandSize::Size64, reg, dst),
943 RegMem::Mem { addr } => Self::mov64_m_r(addr, dst),
944 }
945 }
946
mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst947 pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
948 debug_assert!(src.get_class() == RegClass::I64);
949 Inst::MovRM {
950 size,
951 src,
952 dst: dst.into(),
953 }
954 }
955
lea(addr: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst956 pub(crate) fn lea(addr: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
957 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
958 Inst::LoadEffectiveAddress {
959 addr: addr.into(),
960 dst,
961 }
962 }
963
shift_r( size: OperandSize, kind: ShiftKind, num_bits: Option<u8>, dst: Writable<Reg>, ) -> Inst964 pub(crate) fn shift_r(
965 size: OperandSize,
966 kind: ShiftKind,
967 num_bits: Option<u8>,
968 dst: Writable<Reg>,
969 ) -> Inst {
970 debug_assert!(if let Some(num_bits) = num_bits {
971 num_bits < size.to_bits()
972 } else {
973 true
974 });
975 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
976 Inst::ShiftR {
977 size,
978 kind,
979 num_bits,
980 dst,
981 }
982 }
983
984 /// Does a comparison of dst - src for operands of size `size`, as stated by the machine
985 /// instruction semantics. Be careful with the order of parameters!
cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst986 pub(crate) fn cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {
987 src.assert_regclass_is(RegClass::I64);
988 debug_assert_eq!(dst.get_class(), RegClass::I64);
989 Inst::CmpRmiR {
990 size,
991 src,
992 dst,
993 opcode: CmpOpcode::Cmp,
994 }
995 }
996
997 /// Does a comparison of dst & src for operands of size `size`.
test_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst998 pub(crate) fn test_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {
999 src.assert_regclass_is(RegClass::I64);
1000 debug_assert_eq!(dst.get_class(), RegClass::I64);
1001 Inst::CmpRmiR {
1002 size,
1003 src,
1004 dst,
1005 opcode: CmpOpcode::Test,
1006 }
1007 }
1008
trap(trap_code: TrapCode) -> Inst1009 pub(crate) fn trap(trap_code: TrapCode) -> Inst {
1010 Inst::Ud2 {
1011 trap_code: trap_code,
1012 }
1013 }
1014
setcc(cc: CC, dst: Writable<Reg>) -> Inst1015 pub(crate) fn setcc(cc: CC, dst: Writable<Reg>) -> Inst {
1016 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
1017 Inst::Setcc { cc, dst }
1018 }
1019
cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst1020 pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
1021 debug_assert!(size.is_one_of(&[
1022 OperandSize::Size16,
1023 OperandSize::Size32,
1024 OperandSize::Size64
1025 ]));
1026 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
1027 Inst::Cmove { size, cc, src, dst }
1028 }
1029
xmm_cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst1030 pub(crate) fn xmm_cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
1031 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
1032 src.assert_regclass_is(RegClass::V128);
1033 debug_assert!(dst.to_reg().get_class() == RegClass::V128);
1034 Inst::XmmCmove { size, cc, src, dst }
1035 }
1036
push64(src: RegMemImm) -> Inst1037 pub(crate) fn push64(src: RegMemImm) -> Inst {
1038 src.assert_regclass_is(RegClass::I64);
1039 Inst::Push64 { src }
1040 }
1041
pop64(dst: Writable<Reg>) -> Inst1042 pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
1043 debug_assert!(dst.to_reg().get_class() == RegClass::I64);
1044 Inst::Pop64 { dst }
1045 }
1046
call_known( dest: ExternalName, uses: Vec<Reg>, defs: Vec<Writable<Reg>>, opcode: Opcode, ) -> Inst1047 pub(crate) fn call_known(
1048 dest: ExternalName,
1049 uses: Vec<Reg>,
1050 defs: Vec<Writable<Reg>>,
1051 opcode: Opcode,
1052 ) -> Inst {
1053 Inst::CallKnown {
1054 dest,
1055 uses,
1056 defs,
1057 opcode,
1058 }
1059 }
1060
call_unknown( dest: RegMem, uses: Vec<Reg>, defs: Vec<Writable<Reg>>, opcode: Opcode, ) -> Inst1061 pub(crate) fn call_unknown(
1062 dest: RegMem,
1063 uses: Vec<Reg>,
1064 defs: Vec<Writable<Reg>>,
1065 opcode: Opcode,
1066 ) -> Inst {
1067 dest.assert_regclass_is(RegClass::I64);
1068 Inst::CallUnknown {
1069 dest,
1070 uses,
1071 defs,
1072 opcode,
1073 }
1074 }
1075
ret() -> Inst1076 pub(crate) fn ret() -> Inst {
1077 Inst::Ret
1078 }
1079
epilogue_placeholder() -> Inst1080 pub(crate) fn epilogue_placeholder() -> Inst {
1081 Inst::EpiloguePlaceholder
1082 }
1083
jmp_known(dst: MachLabel) -> Inst1084 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
1085 Inst::JmpKnown { dst }
1086 }
1087
jmp_if(cc: CC, taken: MachLabel) -> Inst1088 pub(crate) fn jmp_if(cc: CC, taken: MachLabel) -> Inst {
1089 Inst::JmpIf { cc, taken }
1090 }
1091
jmp_cond(cc: CC, taken: MachLabel, not_taken: MachLabel) -> Inst1092 pub(crate) fn jmp_cond(cc: CC, taken: MachLabel, not_taken: MachLabel) -> Inst {
1093 Inst::JmpCond {
1094 cc,
1095 taken,
1096 not_taken,
1097 }
1098 }
1099
jmp_unknown(target: RegMem) -> Inst1100 pub(crate) fn jmp_unknown(target: RegMem) -> Inst {
1101 target.assert_regclass_is(RegClass::I64);
1102 Inst::JmpUnknown { target }
1103 }
1104
trap_if(cc: CC, trap_code: TrapCode) -> Inst1105 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
1106 Inst::TrapIf { cc, trap_code }
1107 }
1108
1109 /// Choose which instruction to use for loading a register value from memory. For loads smaller
1110 /// than 64 bits, this method expects a way to extend the value (i.e. [ExtKind::SignExtend],
1111 /// [ExtKind::ZeroExtend]); loads with no extension necessary will ignore this.
load( ty: Type, from_addr: impl Into<SyntheticAmode>, to_reg: Writable<Reg>, ext_kind: ExtKind, ) -> Inst1112 pub(crate) fn load(
1113 ty: Type,
1114 from_addr: impl Into<SyntheticAmode>,
1115 to_reg: Writable<Reg>,
1116 ext_kind: ExtKind,
1117 ) -> Inst {
1118 let rc = to_reg.to_reg().get_class();
1119 match rc {
1120 RegClass::I64 => {
1121 let ext_mode = match ty.bytes() {
1122 1 => Some(ExtMode::BQ),
1123 2 => Some(ExtMode::WQ),
1124 4 => Some(ExtMode::LQ),
1125 8 => None,
1126 _ => unreachable!("the type should never use a scalar load: {}", ty),
1127 };
1128 if let Some(ext_mode) = ext_mode {
1129 // Values smaller than 64 bits must be extended in some way.
1130 match ext_kind {
1131 ExtKind::SignExtend => {
1132 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
1133 }
1134 ExtKind::ZeroExtend => {
1135 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
1136 }
1137 ExtKind::None => panic!(
1138 "expected an extension kind for extension mode: {:?}",
1139 ext_mode
1140 ),
1141 }
1142 } else {
1143 // 64-bit values can be moved directly.
1144 Inst::mov64_m_r(from_addr, to_reg)
1145 }
1146 }
1147 RegClass::V128 => {
1148 let opcode = match ty {
1149 types::F32 => SseOpcode::Movss,
1150 types::F64 => SseOpcode::Movsd,
1151 types::F32X4 => SseOpcode::Movups,
1152 types::F64X2 => SseOpcode::Movupd,
1153 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
1154 _ => unimplemented!("unable to load type: {}", ty),
1155 };
1156 Inst::xmm_unary_rm_r(opcode, RegMem::mem(from_addr), to_reg)
1157 }
1158 _ => panic!("unable to generate load for register class: {:?}", rc),
1159 }
1160 }
1161
1162 /// Choose which instruction to use for storing a register value to memory.
store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst1163 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
1164 let rc = from_reg.get_class();
1165 match rc {
1166 RegClass::I64 => Inst::mov_r_m(OperandSize::from_ty(ty), from_reg, to_addr),
1167 RegClass::V128 => {
1168 let opcode = match ty {
1169 types::F32 => SseOpcode::Movss,
1170 types::F64 => SseOpcode::Movsd,
1171 types::F32X4 => SseOpcode::Movups,
1172 types::F64X2 => SseOpcode::Movupd,
1173 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
1174 _ => unimplemented!("unable to store type: {}", ty),
1175 };
1176 Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
1177 }
1178 _ => panic!("unable to generate store for register class: {:?}", rc),
1179 }
1180 }
1181 }
1182
1183 // Inst helpers.
1184
1185 impl Inst {
1186 /// In certain cases, instructions of this format can act as a definition of an XMM register,
1187 /// producing a value that is independent of its initial value.
1188 ///
1189 /// For example, a vector equality comparison (`cmppd` or `cmpps`) that compares a register to
1190 /// itself will generate all ones as a result, regardless of its value. From the register
1191 /// allocator's point of view, we should (i) record the first register, which is normally a
1192 /// mod, as a def instead; and (ii) not record the second register as a use, because it is the
1193 /// same as the first register (already handled).
produces_const(&self) -> bool1194 fn produces_const(&self) -> bool {
1195 match self {
1196 Self::AluRmiR { op, src, dst, .. } => {
1197 src.to_reg() == Some(dst.to_reg())
1198 && (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub)
1199 }
1200
1201 Self::XmmRmR { op, src, dst, .. } => {
1202 src.to_reg() == Some(dst.to_reg())
1203 && (*op == SseOpcode::Xorps
1204 || *op == SseOpcode::Xorpd
1205 || *op == SseOpcode::Pxor
1206 || *op == SseOpcode::Pcmpeqb
1207 || *op == SseOpcode::Pcmpeqw
1208 || *op == SseOpcode::Pcmpeqd
1209 || *op == SseOpcode::Pcmpeqq)
1210 }
1211
1212 Self::XmmRmRImm {
1213 op, src, dst, imm, ..
1214 } => {
1215 src.to_reg() == Some(dst.to_reg())
1216 && (*op == SseOpcode::Cmppd || *op == SseOpcode::Cmpps)
1217 && *imm == FcmpImm::Equal.encode()
1218 }
1219
1220 _ => false,
1221 }
1222 }
1223
1224 /// Choose which instruction to use for comparing two values for equality.
equals(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst1225 pub(crate) fn equals(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
1226 match ty {
1227 types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to),
1228 types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to),
1229 types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to),
1230 types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to),
1231 types::F32X4 => Inst::xmm_rm_r_imm(
1232 SseOpcode::Cmpps,
1233 from,
1234 to,
1235 FcmpImm::Equal.encode(),
1236 OperandSize::Size32,
1237 ),
1238 types::F64X2 => Inst::xmm_rm_r_imm(
1239 SseOpcode::Cmppd,
1240 from,
1241 to,
1242 FcmpImm::Equal.encode(),
1243 OperandSize::Size32,
1244 ),
1245 _ => unimplemented!("unimplemented type for Inst::equals: {}", ty),
1246 }
1247 }
1248
1249 /// Choose which instruction to use for computing a bitwise AND on two values.
and(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst1250 pub(crate) fn and(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
1251 match ty {
1252 types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andps, from, to),
1253 types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andpd, from, to),
1254 _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pand, from, to),
1255 _ => unimplemented!("unimplemented type for Inst::and: {}", ty),
1256 }
1257 }
1258
1259 /// Choose which instruction to use for computing a bitwise AND NOT on two values.
and_not(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst1260 pub(crate) fn and_not(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
1261 match ty {
1262 types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andnps, from, to),
1263 types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andnpd, from, to),
1264 _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pandn, from, to),
1265 _ => unimplemented!("unimplemented type for Inst::and_not: {}", ty),
1266 }
1267 }
1268
1269 /// Choose which instruction to use for computing a bitwise OR on two values.
or(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst1270 pub(crate) fn or(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
1271 match ty {
1272 types::F32X4 => Inst::xmm_rm_r(SseOpcode::Orps, from, to),
1273 types::F64X2 => Inst::xmm_rm_r(SseOpcode::Orpd, from, to),
1274 _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Por, from, to),
1275 _ => unimplemented!("unimplemented type for Inst::or: {}", ty),
1276 }
1277 }
1278
1279 /// Choose which instruction to use for computing a bitwise XOR on two values.
xor(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst1280 pub(crate) fn xor(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
1281 match ty {
1282 types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to),
1283 types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to),
1284 _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pxor, from, to),
1285 _ => unimplemented!("unimplemented type for Inst::xor: {}", ty),
1286 }
1287 }
1288 }
1289
1290 //=============================================================================
1291 // Instructions: printing
1292
1293 impl PrettyPrint for Inst {
show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String1294 fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
1295 fn ljustify(s: String) -> String {
1296 let w = 7;
1297 if s.len() >= w {
1298 s
1299 } else {
1300 let need = usize::min(w, w - s.len());
1301 s + &format!("{nil: <width$}", nil = "", width = need)
1302 }
1303 }
1304
1305 fn ljustify2(s1: String, s2: String) -> String {
1306 ljustify(s1 + &s2)
1307 }
1308
1309 fn suffix_lq(size: OperandSize) -> String {
1310 match size {
1311 OperandSize::Size32 => "l",
1312 OperandSize::Size64 => "q",
1313 _ => unreachable!(),
1314 }
1315 .to_string()
1316 }
1317
1318 fn suffix_lqb(size: OperandSize, is_8: bool) -> String {
1319 match (size, is_8) {
1320 (_, true) => "b",
1321 (OperandSize::Size32, false) => "l",
1322 (OperandSize::Size64, false) => "q",
1323 _ => unreachable!(),
1324 }
1325 .to_string()
1326 }
1327
1328 fn size_lqb(size: OperandSize, is_8: bool) -> u8 {
1329 if is_8 {
1330 return 1;
1331 }
1332 size.to_bytes()
1333 }
1334
1335 fn suffix_bwlq(size: OperandSize) -> String {
1336 match size {
1337 OperandSize::Size8 => "b".to_string(),
1338 OperandSize::Size16 => "w".to_string(),
1339 OperandSize::Size32 => "l".to_string(),
1340 OperandSize::Size64 => "q".to_string(),
1341 }
1342 }
1343
1344 match self {
1345 Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len),
1346
1347 Inst::AluRmiR { size, op, src, dst } => format!(
1348 "{} {}, {}",
1349 ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
1350 src.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
1351 show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
1352 ),
1353
1354 Inst::UnaryRmR { src, dst, op, size } => format!(
1355 "{} {}, {}",
1356 ljustify2(op.to_string(), suffix_bwlq(*size)),
1357 src.show_rru_sized(mb_rru, size.to_bytes()),
1358 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()),
1359 ),
1360
1361 Inst::Not { size, src } => format!(
1362 "{} {}",
1363 ljustify2("not".to_string(), suffix_bwlq(*size)),
1364 show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes())
1365 ),
1366
1367 Inst::Neg { size, src } => format!(
1368 "{} {}",
1369 ljustify2("neg".to_string(), suffix_bwlq(*size)),
1370 show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes())
1371 ),
1372
1373 Inst::Div {
1374 size,
1375 signed,
1376 divisor,
1377 ..
1378 } => format!(
1379 "{} {}",
1380 ljustify(if *signed {
1381 "idiv".to_string()
1382 } else {
1383 "div".into()
1384 }),
1385 divisor.show_rru_sized(mb_rru, size.to_bytes())
1386 ),
1387
1388 Inst::MulHi {
1389 size, signed, rhs, ..
1390 } => format!(
1391 "{} {}",
1392 ljustify(if *signed {
1393 "imul".to_string()
1394 } else {
1395 "mul".to_string()
1396 }),
1397 rhs.show_rru_sized(mb_rru, size.to_bytes())
1398 ),
1399
1400 Inst::CheckedDivOrRemSeq {
1401 kind,
1402 size,
1403 divisor,
1404 ..
1405 } => format!(
1406 "{} $rax:$rdx, {}",
1407 match kind {
1408 DivOrRemKind::SignedDiv => "sdiv",
1409 DivOrRemKind::UnsignedDiv => "udiv",
1410 DivOrRemKind::SignedRem => "srem",
1411 DivOrRemKind::UnsignedRem => "urem",
1412 },
1413 show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()),
1414 ),
1415
1416 Inst::SignExtendData { size } => match size {
1417 OperandSize::Size8 => "cbw",
1418 OperandSize::Size16 => "cwd",
1419 OperandSize::Size32 => "cdq",
1420 OperandSize::Size64 => "cqo",
1421 }
1422 .into(),
1423
1424 Inst::XmmUnaryRmR { op, src, dst, .. } => format!(
1425 "{} {}, {}",
1426 ljustify(op.to_string()),
1427 src.show_rru_sized(mb_rru, op.src_size()),
1428 show_ireg_sized(dst.to_reg(), mb_rru, 8),
1429 ),
1430
1431 Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!(
1432 "{} {}, {}",
1433 ljustify(op.to_string()),
1434 src.show_rru_sized(mb_rru, 8),
1435 show_ireg_sized(dst.to_reg(), mb_rru, 8),
1436 ),
1437
1438 Inst::XmmMovRM { op, src, dst, .. } => format!(
1439 "{} {}, {}",
1440 ljustify(op.to_string()),
1441 show_ireg_sized(*src, mb_rru, 8),
1442 dst.show_rru(mb_rru),
1443 ),
1444
1445 Inst::XmmRmR { op, src, dst, .. } => format!(
1446 "{} {}, {}",
1447 ljustify(op.to_string()),
1448 src.show_rru_sized(mb_rru, 8),
1449 show_ireg_sized(dst.to_reg(), mb_rru, 8),
1450 ),
1451
1452 Inst::XmmRmREvex {
1453 op,
1454 src1,
1455 src2,
1456 dst,
1457 ..
1458 } => format!(
1459 "{} {}, {}, {}",
1460 ljustify(op.to_string()),
1461 src1.show_rru_sized(mb_rru, 8),
1462 show_ireg_sized(*src2, mb_rru, 8),
1463 show_ireg_sized(dst.to_reg(), mb_rru, 8),
1464 ),
1465
1466 Inst::XmmMinMaxSeq {
1467 lhs,
1468 rhs_dst,
1469 is_min,
1470 size,
1471 } => format!(
1472 "{} {}, {}",
1473 ljustify2(
1474 if *is_min {
1475 "xmm min seq ".to_string()
1476 } else {
1477 "xmm max seq ".to_string()
1478 },
1479 format!("f{}", size.to_bits())
1480 ),
1481 show_ireg_sized(*lhs, mb_rru, 8),
1482 show_ireg_sized(rhs_dst.to_reg(), mb_rru, 8),
1483 ),
1484
1485 Inst::XmmRmRImm {
1486 op,
1487 src,
1488 dst,
1489 imm,
1490 size,
1491 ..
1492 } => format!(
1493 "{} ${}, {}, {}",
1494 ljustify(format!(
1495 "{}{}",
1496 op.to_string(),
1497 if *size == OperandSize::Size64 {
1498 ".w"
1499 } else {
1500 ""
1501 }
1502 )),
1503 imm,
1504 src.show_rru(mb_rru),
1505 dst.show_rru(mb_rru),
1506 ),
1507
1508 Inst::XmmUninitializedValue { dst } => {
1509 format!("{} {}", ljustify("uninit".into()), dst.show_rru(mb_rru),)
1510 }
1511
1512 Inst::XmmLoadConst { src, dst, .. } => {
1513 format!("load_const {:?}, {}", src, dst.show_rru(mb_rru),)
1514 }
1515
1516 Inst::XmmToGpr {
1517 op,
1518 src,
1519 dst,
1520 dst_size,
1521 } => {
1522 let dst_size = dst_size.to_bytes();
1523 format!(
1524 "{} {}, {}",
1525 ljustify(op.to_string()),
1526 src.show_rru(mb_rru),
1527 show_ireg_sized(dst.to_reg(), mb_rru, dst_size),
1528 )
1529 }
1530
1531 Inst::GprToXmm {
1532 op,
1533 src,
1534 src_size,
1535 dst,
1536 } => format!(
1537 "{} {}, {}",
1538 ljustify(op.to_string()),
1539 src.show_rru_sized(mb_rru, src_size.to_bytes()),
1540 dst.show_rru(mb_rru)
1541 ),
1542
1543 Inst::XmmCmpRmR { op, src, dst } => format!(
1544 "{} {}, {}",
1545 ljustify(op.to_string()),
1546 src.show_rru_sized(mb_rru, 8),
1547 show_ireg_sized(*dst, mb_rru, 8),
1548 ),
1549
1550 Inst::CvtUint64ToFloatSeq {
1551 src, dst, dst_size, ..
1552 } => format!(
1553 "{} {}, {}",
1554 ljustify(format!(
1555 "u64_to_{}_seq",
1556 if *dst_size == OperandSize::Size64 {
1557 "f64"
1558 } else {
1559 "f32"
1560 }
1561 )),
1562 show_ireg_sized(src.to_reg(), mb_rru, 8),
1563 dst.show_rru(mb_rru),
1564 ),
1565
1566 Inst::CvtFloatToSintSeq {
1567 src,
1568 dst,
1569 src_size,
1570 dst_size,
1571 ..
1572 } => format!(
1573 "{} {}, {}",
1574 ljustify(format!(
1575 "cvt_float{}_to_sint{}_seq",
1576 src_size.to_bits(),
1577 dst_size.to_bits()
1578 )),
1579 show_ireg_sized(src.to_reg(), mb_rru, 8),
1580 show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()),
1581 ),
1582
1583 Inst::CvtFloatToUintSeq {
1584 src,
1585 dst,
1586 src_size,
1587 dst_size,
1588 ..
1589 } => format!(
1590 "{} {}, {}",
1591 ljustify(format!(
1592 "cvt_float{}_to_uint{}_seq",
1593 src_size.to_bits(),
1594 dst_size.to_bits()
1595 )),
1596 show_ireg_sized(src.to_reg(), mb_rru, 8),
1597 show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()),
1598 ),
1599
1600 Inst::Imm {
1601 dst_size,
1602 simm64,
1603 dst,
1604 } => {
1605 if *dst_size == OperandSize::Size64 {
1606 format!(
1607 "{} ${}, {}",
1608 ljustify("movabsq".to_string()),
1609 *simm64 as i64,
1610 show_ireg_sized(dst.to_reg(), mb_rru, 8)
1611 )
1612 } else {
1613 format!(
1614 "{} ${}, {}",
1615 ljustify("movl".to_string()),
1616 (*simm64 as u32) as i32,
1617 show_ireg_sized(dst.to_reg(), mb_rru, 4)
1618 )
1619 }
1620 }
1621
1622 Inst::MovRR { size, src, dst } => format!(
1623 "{} {}, {}",
1624 ljustify2("mov".to_string(), suffix_lq(*size)),
1625 show_ireg_sized(*src, mb_rru, size.to_bytes()),
1626 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
1627 ),
1628
1629 Inst::MovzxRmR {
1630 ext_mode, src, dst, ..
1631 } => {
1632 if *ext_mode == ExtMode::LQ {
1633 format!(
1634 "{} {}, {}",
1635 ljustify("movl".to_string()),
1636 src.show_rru_sized(mb_rru, ext_mode.src_size()),
1637 show_ireg_sized(dst.to_reg(), mb_rru, 4)
1638 )
1639 } else {
1640 format!(
1641 "{} {}, {}",
1642 ljustify2("movz".to_string(), ext_mode.to_string()),
1643 src.show_rru_sized(mb_rru, ext_mode.src_size()),
1644 show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
1645 )
1646 }
1647 }
1648
1649 Inst::Mov64MR { src, dst, .. } => format!(
1650 "{} {}, {}",
1651 ljustify("movq".to_string()),
1652 src.show_rru(mb_rru),
1653 dst.show_rru(mb_rru)
1654 ),
1655
1656 Inst::LoadEffectiveAddress { addr, dst } => format!(
1657 "{} {}, {}",
1658 ljustify("lea".to_string()),
1659 addr.show_rru(mb_rru),
1660 dst.show_rru(mb_rru)
1661 ),
1662
1663 Inst::MovsxRmR {
1664 ext_mode, src, dst, ..
1665 } => format!(
1666 "{} {}, {}",
1667 ljustify2("movs".to_string(), ext_mode.to_string()),
1668 src.show_rru_sized(mb_rru, ext_mode.src_size()),
1669 show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
1670 ),
1671
1672 Inst::MovRM { size, src, dst, .. } => format!(
1673 "{} {}, {}",
1674 ljustify2("mov".to_string(), suffix_bwlq(*size)),
1675 show_ireg_sized(*src, mb_rru, size.to_bytes()),
1676 dst.show_rru(mb_rru)
1677 ),
1678
1679 Inst::ShiftR {
1680 size,
1681 kind,
1682 num_bits,
1683 dst,
1684 } => match num_bits {
1685 None => format!(
1686 "{} %cl, {}",
1687 ljustify2(kind.to_string(), suffix_bwlq(*size)),
1688 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
1689 ),
1690
1691 Some(num_bits) => format!(
1692 "{} ${}, {}",
1693 ljustify2(kind.to_string(), suffix_bwlq(*size)),
1694 num_bits,
1695 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
1696 ),
1697 },
1698
1699 Inst::XmmRmiReg { opcode, src, dst } => format!(
1700 "{} {}, {}",
1701 ljustify(opcode.to_string()),
1702 src.show_rru(mb_rru),
1703 dst.to_reg().show_rru(mb_rru)
1704 ),
1705
1706 Inst::CmpRmiR {
1707 size,
1708 src,
1709 dst,
1710 opcode,
1711 } => {
1712 let op = match opcode {
1713 CmpOpcode::Cmp => "cmp",
1714 CmpOpcode::Test => "test",
1715 };
1716 format!(
1717 "{} {}, {}",
1718 ljustify2(op.to_string(), suffix_bwlq(*size)),
1719 src.show_rru_sized(mb_rru, size.to_bytes()),
1720 show_ireg_sized(*dst, mb_rru, size.to_bytes())
1721 )
1722 }
1723
1724 Inst::Setcc { cc, dst } => format!(
1725 "{} {}",
1726 ljustify2("set".to_string(), cc.to_string()),
1727 show_ireg_sized(dst.to_reg(), mb_rru, 1)
1728 ),
1729
1730 Inst::Cmove { size, cc, src, dst } => format!(
1731 "{} {}, {}",
1732 ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
1733 src.show_rru_sized(mb_rru, size.to_bytes()),
1734 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
1735 ),
1736
1737 Inst::XmmCmove { size, cc, src, dst } => {
1738 format!(
1739 "j{} $next; mov{} {}, {}; $next: ",
1740 cc.invert().to_string(),
1741 if *size == OperandSize::Size64 {
1742 "sd"
1743 } else {
1744 "ss"
1745 },
1746 src.show_rru_sized(mb_rru, size.to_bytes()),
1747 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
1748 )
1749 }
1750
1751 Inst::Push64 { src } => {
1752 format!("{} {}", ljustify("pushq".to_string()), src.show_rru(mb_rru))
1753 }
1754
1755 Inst::Pop64 { dst } => {
1756 format!("{} {}", ljustify("popq".to_string()), dst.show_rru(mb_rru))
1757 }
1758
1759 Inst::CallKnown { dest, .. } => format!("{} {:?}", ljustify("call".to_string()), dest),
1760
1761 Inst::CallUnknown { dest, .. } => format!(
1762 "{} *{}",
1763 ljustify("call".to_string()),
1764 dest.show_rru(mb_rru)
1765 ),
1766
1767 Inst::Ret => "ret".to_string(),
1768
1769 Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(),
1770
1771 Inst::JmpKnown { dst } => {
1772 format!("{} {}", ljustify("jmp".to_string()), dst.to_string())
1773 }
1774
1775 Inst::JmpIf { cc, taken } => format!(
1776 "{} {}",
1777 ljustify2("j".to_string(), cc.to_string()),
1778 taken.to_string(),
1779 ),
1780
1781 Inst::JmpCond {
1782 cc,
1783 taken,
1784 not_taken,
1785 } => format!(
1786 "{} {}; j {}",
1787 ljustify2("j".to_string(), cc.to_string()),
1788 taken.to_string(),
1789 not_taken.to_string()
1790 ),
1791
1792 Inst::JmpTableSeq { idx, .. } => {
1793 format!("{} {}", ljustify("br_table".into()), idx.show_rru(mb_rru))
1794 }
1795
1796 Inst::JmpUnknown { target } => format!(
1797 "{} *{}",
1798 ljustify("jmp".to_string()),
1799 target.show_rru(mb_rru)
1800 ),
1801
1802 Inst::TrapIf { cc, trap_code, .. } => {
1803 format!("j{} ; ud2 {} ;", cc.invert().to_string(), trap_code)
1804 }
1805
1806 Inst::LoadExtName {
1807 dst, name, offset, ..
1808 } => format!(
1809 "{} {}+{}, {}",
1810 ljustify("load_ext_name".into()),
1811 name,
1812 offset,
1813 show_ireg_sized(dst.to_reg(), mb_rru, 8),
1814 ),
1815
1816 Inst::LockCmpxchg { ty, src, dst, .. } => {
1817 let size = ty.bytes() as u8;
1818 format!(
1819 "lock cmpxchg{} {}, {}",
1820 suffix_bwlq(OperandSize::from_bytes(size as u32)),
1821 show_ireg_sized(*src, mb_rru, size),
1822 dst.show_rru(mb_rru)
1823 )
1824 }
1825
1826 Inst::AtomicRmwSeq { ty, op, .. } => {
1827 format!(
1828 "atomically {{ {}_bits_at_[%r9]) {:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}",
1829 ty.bits(), op)
1830 }
1831
1832 Inst::Fence { kind } => match kind {
1833 FenceKind::MFence => "mfence".to_string(),
1834 FenceKind::LFence => "lfence".to_string(),
1835 FenceKind::SFence => "sfence".to_string(),
1836 },
1837
1838 Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset),
1839
1840 Inst::Hlt => "hlt".into(),
1841
1842 Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
1843
1844 Inst::ElfTlsGetAddr { ref symbol } => {
1845 format!("elf_tls_get_addr {:?}", symbol)
1846 }
1847
1848 Inst::MachOTlsGetAddr { ref symbol } => {
1849 format!("macho_tls_get_addr {:?}", symbol)
1850 }
1851
1852 Inst::ValueLabelMarker { label, reg } => {
1853 format!("value_label {:?}, {}", label, reg.show_rru(mb_rru))
1854 }
1855
1856 Inst::Unwind { inst } => {
1857 format!("unwind {:?}", inst)
1858 }
1859 }
1860 }
1861 }
1862
1863 // Temp hook for legacy printing machinery
1864 impl fmt::Debug for Inst {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result1865 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1866 // Print the insn without a Universe :-(
1867 write!(fmt, "{}", self.show_rru(None))
1868 }
1869 }
1870
x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector)1871 fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
1872 // This is a bit subtle. If some register is in the modified set, then it may not be in either
1873 // the use or def sets. However, enforcing that directly is somewhat difficult. Instead,
1874 // regalloc.rs will "fix" this for us by removing the modified set from the use and def
1875 // sets.
1876 match inst {
1877 Inst::AluRmiR { src, dst, .. } => {
1878 if inst.produces_const() {
1879 // No need to account for src, since src == dst.
1880 collector.add_def(*dst);
1881 } else {
1882 src.get_regs_as_uses(collector);
1883 collector.add_mod(*dst);
1884 }
1885 }
1886 Inst::Not { src, .. } => {
1887 collector.add_mod(*src);
1888 }
1889 Inst::Neg { src, .. } => {
1890 collector.add_mod(*src);
1891 }
1892 Inst::Div { size, divisor, .. } => {
1893 collector.add_mod(Writable::from_reg(regs::rax()));
1894 if *size == OperandSize::Size8 {
1895 collector.add_def(Writable::from_reg(regs::rdx()));
1896 } else {
1897 collector.add_mod(Writable::from_reg(regs::rdx()));
1898 }
1899 divisor.get_regs_as_uses(collector);
1900 }
1901 Inst::MulHi { rhs, .. } => {
1902 collector.add_mod(Writable::from_reg(regs::rax()));
1903 collector.add_def(Writable::from_reg(regs::rdx()));
1904 rhs.get_regs_as_uses(collector);
1905 }
1906 Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
1907 // Mark both fixed registers as mods, to avoid an early clobber problem in codegen
1908 // (i.e. the temporary is allocated one of the fixed registers). This requires writing
1909 // the rdx register *before* the instruction, which is not too bad.
1910 collector.add_mod(Writable::from_reg(regs::rax()));
1911 collector.add_mod(Writable::from_reg(regs::rdx()));
1912 collector.add_mod(*divisor);
1913 if let Some(tmp) = tmp {
1914 collector.add_def(*tmp);
1915 }
1916 }
1917 Inst::SignExtendData { size } => match size {
1918 OperandSize::Size8 => collector.add_mod(Writable::from_reg(regs::rax())),
1919 _ => {
1920 collector.add_use(regs::rax());
1921 collector.add_def(Writable::from_reg(regs::rdx()));
1922 }
1923 },
1924 Inst::UnaryRmR { src, dst, .. }
1925 | Inst::XmmUnaryRmR { src, dst, .. }
1926 | Inst::XmmUnaryRmREvex { src, dst, .. } => {
1927 src.get_regs_as_uses(collector);
1928 collector.add_def(*dst);
1929 }
1930 Inst::XmmRmR { src, dst, op, .. } => {
1931 if inst.produces_const() {
1932 // No need to account for src, since src == dst.
1933 collector.add_def(*dst);
1934 } else {
1935 src.get_regs_as_uses(collector);
1936 collector.add_mod(*dst);
1937 // Some instructions have an implicit use of XMM0.
1938 if *op == SseOpcode::Blendvpd
1939 || *op == SseOpcode::Blendvps
1940 || *op == SseOpcode::Pblendvb
1941 {
1942 collector.add_use(regs::xmm0());
1943 }
1944 }
1945 }
1946 Inst::XmmRmREvex {
1947 src1, src2, dst, ..
1948 } => {
1949 src1.get_regs_as_uses(collector);
1950 collector.add_use(*src2);
1951 collector.add_def(*dst);
1952 }
1953 Inst::XmmRmRImm { op, src, dst, .. } => {
1954 if inst.produces_const() {
1955 // No need to account for src, since src == dst.
1956 collector.add_def(*dst);
1957 } else if *op == SseOpcode::Pextrb
1958 || *op == SseOpcode::Pextrw
1959 || *op == SseOpcode::Pextrd
1960 || *op == SseOpcode::Pshufd
1961 || *op == SseOpcode::Roundss
1962 || *op == SseOpcode::Roundsd
1963 || *op == SseOpcode::Roundps
1964 || *op == SseOpcode::Roundpd
1965 {
1966 src.get_regs_as_uses(collector);
1967 collector.add_def(*dst);
1968 } else {
1969 src.get_regs_as_uses(collector);
1970 collector.add_mod(*dst);
1971 }
1972 }
1973 Inst::XmmUninitializedValue { dst } => collector.add_def(*dst),
1974 Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst),
1975 Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => {
1976 collector.add_use(*lhs);
1977 collector.add_mod(*rhs_dst);
1978 }
1979 Inst::XmmRmiReg { src, dst, .. } => {
1980 src.get_regs_as_uses(collector);
1981 collector.add_mod(*dst);
1982 }
1983 Inst::XmmMovRM { src, dst, .. } => {
1984 collector.add_use(*src);
1985 dst.get_regs_as_uses(collector);
1986 }
1987 Inst::XmmCmpRmR { src, dst, .. } => {
1988 src.get_regs_as_uses(collector);
1989 collector.add_use(*dst);
1990 }
1991 Inst::Imm { dst, .. } => {
1992 collector.add_def(*dst);
1993 }
1994 Inst::MovRR { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => {
1995 collector.add_use(*src);
1996 collector.add_def(*dst);
1997 }
1998 Inst::GprToXmm { src, dst, .. } => {
1999 src.get_regs_as_uses(collector);
2000 collector.add_def(*dst);
2001 }
2002 Inst::CvtUint64ToFloatSeq {
2003 src,
2004 dst,
2005 tmp_gpr1,
2006 tmp_gpr2,
2007 ..
2008 } => {
2009 collector.add_mod(*src);
2010 collector.add_def(*dst);
2011 collector.add_def(*tmp_gpr1);
2012 collector.add_def(*tmp_gpr2);
2013 }
2014 Inst::CvtFloatToSintSeq {
2015 src,
2016 dst,
2017 tmp_xmm,
2018 tmp_gpr,
2019 ..
2020 }
2021 | Inst::CvtFloatToUintSeq {
2022 src,
2023 dst,
2024 tmp_gpr,
2025 tmp_xmm,
2026 ..
2027 } => {
2028 collector.add_mod(*src);
2029 collector.add_def(*dst);
2030 collector.add_def(*tmp_gpr);
2031 collector.add_def(*tmp_xmm);
2032 }
2033 Inst::MovzxRmR { src, dst, .. } => {
2034 src.get_regs_as_uses(collector);
2035 collector.add_def(*dst);
2036 }
2037 Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => {
2038 src.get_regs_as_uses(collector);
2039 collector.add_def(*dst)
2040 }
2041 Inst::MovsxRmR { src, dst, .. } => {
2042 src.get_regs_as_uses(collector);
2043 collector.add_def(*dst);
2044 }
2045 Inst::MovRM { src, dst, .. } => {
2046 collector.add_use(*src);
2047 dst.get_regs_as_uses(collector);
2048 }
2049 Inst::ShiftR { num_bits, dst, .. } => {
2050 if num_bits.is_none() {
2051 collector.add_use(regs::rcx());
2052 }
2053 collector.add_mod(*dst);
2054 }
2055 Inst::CmpRmiR { src, dst, .. } => {
2056 src.get_regs_as_uses(collector);
2057 collector.add_use(*dst); // yes, really `add_use`
2058 }
2059 Inst::Setcc { dst, .. } => {
2060 collector.add_def(*dst);
2061 }
2062 Inst::Cmove { src, dst, .. } | Inst::XmmCmove { src, dst, .. } => {
2063 src.get_regs_as_uses(collector);
2064 collector.add_mod(*dst);
2065 }
2066 Inst::Push64 { src } => {
2067 src.get_regs_as_uses(collector);
2068 collector.add_mod(Writable::from_reg(regs::rsp()));
2069 }
2070 Inst::Pop64 { dst } => {
2071 collector.add_def(*dst);
2072 }
2073
2074 Inst::CallKnown {
2075 ref uses, ref defs, ..
2076 } => {
2077 collector.add_uses(uses);
2078 collector.add_defs(defs);
2079 }
2080
2081 Inst::CallUnknown {
2082 ref uses,
2083 ref defs,
2084 dest,
2085 ..
2086 } => {
2087 collector.add_uses(uses);
2088 collector.add_defs(defs);
2089 dest.get_regs_as_uses(collector);
2090 }
2091
2092 Inst::JmpTableSeq {
2093 ref idx,
2094 ref tmp1,
2095 ref tmp2,
2096 ..
2097 } => {
2098 collector.add_use(*idx);
2099 collector.add_def(*tmp1);
2100 collector.add_def(*tmp2);
2101 }
2102
2103 Inst::JmpUnknown { target } => {
2104 target.get_regs_as_uses(collector);
2105 }
2106
2107 Inst::LoadExtName { dst, .. } => {
2108 collector.add_def(*dst);
2109 }
2110
2111 Inst::LockCmpxchg { src, dst, .. } => {
2112 dst.get_regs_as_uses(collector);
2113 collector.add_use(*src);
2114 collector.add_mod(Writable::from_reg(regs::rax()));
2115 }
2116
2117 Inst::AtomicRmwSeq { .. } => {
2118 collector.add_use(regs::r9());
2119 collector.add_use(regs::r10());
2120 collector.add_def(Writable::from_reg(regs::r11()));
2121 collector.add_def(Writable::from_reg(regs::rax()));
2122 }
2123
2124 Inst::Ret
2125 | Inst::EpiloguePlaceholder
2126 | Inst::JmpKnown { .. }
2127 | Inst::JmpIf { .. }
2128 | Inst::JmpCond { .. }
2129 | Inst::Nop { .. }
2130 | Inst::TrapIf { .. }
2131 | Inst::VirtualSPOffsetAdj { .. }
2132 | Inst::Hlt
2133 | Inst::Ud2 { .. }
2134 | Inst::Fence { .. } => {
2135 // No registers are used.
2136 }
2137
2138 Inst::ElfTlsGetAddr { .. } | Inst::MachOTlsGetAddr { .. } => {
2139 // All caller-saves are clobbered.
2140 //
2141 // We use the SysV calling convention here because the
2142 // pseudoinstruction (and relocation that it emits) is specific to
2143 // ELF systems; other x86-64 targets with other conventions (i.e.,
2144 // Windows) use different TLS strategies.
2145 for reg in X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV) {
2146 collector.add_def(reg);
2147 }
2148 }
2149
2150 Inst::ValueLabelMarker { reg, .. } => {
2151 collector.add_use(*reg);
2152 }
2153
2154 Inst::Unwind { .. } => {}
2155 }
2156 }
2157
2158 //=============================================================================
2159 // Instructions and subcomponents: map_regs
2160
map_use<RUM: RegUsageMapper>(m: &RUM, r: &mut Reg)2161 fn map_use<RUM: RegUsageMapper>(m: &RUM, r: &mut Reg) {
2162 if let Some(reg) = r.as_virtual_reg() {
2163 let new = m.get_use(reg).unwrap().to_reg();
2164 *r = new;
2165 }
2166 }
2167
map_def<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>)2168 fn map_def<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
2169 if let Some(reg) = r.to_reg().as_virtual_reg() {
2170 let new = m.get_def(reg).unwrap().to_reg();
2171 *r = Writable::from_reg(new);
2172 }
2173 }
2174
map_mod<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>)2175 fn map_mod<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
2176 if let Some(reg) = r.to_reg().as_virtual_reg() {
2177 let new = m.get_mod(reg).unwrap().to_reg();
2178 *r = Writable::from_reg(new);
2179 }
2180 }
2181
2182 impl Amode {
map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM)2183 fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
2184 match self {
2185 Amode::ImmReg { ref mut base, .. } => map_use(map, base),
2186 Amode::ImmRegRegShift {
2187 ref mut base,
2188 ref mut index,
2189 ..
2190 } => {
2191 map_use(map, base);
2192 map_use(map, index);
2193 }
2194 Amode::RipRelative { .. } => {
2195 // RIP isn't involved in regalloc.
2196 }
2197 }
2198 }
2199
2200 /// Offset the amode by a fixed offset.
offset(&self, offset: u32) -> Self2201 pub(crate) fn offset(&self, offset: u32) -> Self {
2202 let mut ret = self.clone();
2203 match &mut ret {
2204 &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
2205 &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
2206 _ => panic!("Cannot offset amode: {:?}", self),
2207 }
2208 ret
2209 }
2210 }
2211
2212 impl RegMemImm {
map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM)2213 fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
2214 match self {
2215 RegMemImm::Reg { ref mut reg } => map_use(map, reg),
2216 RegMemImm::Mem { ref mut addr } => addr.map_uses(map),
2217 RegMemImm::Imm { .. } => {}
2218 }
2219 }
2220
map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM)2221 fn map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
2222 match self {
2223 Self::Reg { reg } => {
2224 let mut writable_src = Writable::from_reg(*reg);
2225 map_def(mapper, &mut writable_src);
2226 *self = Self::reg(writable_src.to_reg());
2227 }
2228 _ => panic!("unexpected RegMemImm kind in map_src_reg_as_def"),
2229 }
2230 }
2231 }
2232
2233 impl RegMem {
map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM)2234 fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
2235 match self {
2236 RegMem::Reg { ref mut reg } => map_use(map, reg),
2237 RegMem::Mem { ref mut addr, .. } => addr.map_uses(map),
2238 }
2239 }
2240
map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM)2241 fn map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
2242 match self {
2243 Self::Reg { reg } => {
2244 let mut writable_src = Writable::from_reg(*reg);
2245 map_def(mapper, &mut writable_src);
2246 *self = Self::reg(writable_src.to_reg());
2247 }
2248 _ => panic!("unexpected RegMem kind in map_src_reg_as_def"),
2249 }
2250 }
2251 }
2252
x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM)2253 fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
2254 // Note this must be carefully synchronized with x64_get_regs.
2255 let produces_const = inst.produces_const();
2256
2257 match inst {
2258 // ** Nop
2259 Inst::AluRmiR {
2260 ref mut src,
2261 ref mut dst,
2262 ..
2263 } => {
2264 if produces_const {
2265 src.map_as_def(mapper);
2266 map_def(mapper, dst);
2267 } else {
2268 src.map_uses(mapper);
2269 map_mod(mapper, dst);
2270 }
2271 }
2272 Inst::Not { src, .. } | Inst::Neg { src, .. } => map_mod(mapper, src),
2273 Inst::Div { divisor, .. } => divisor.map_uses(mapper),
2274 Inst::MulHi { rhs, .. } => rhs.map_uses(mapper),
2275 Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
2276 map_mod(mapper, divisor);
2277 if let Some(tmp) = tmp {
2278 map_def(mapper, tmp)
2279 }
2280 }
2281 Inst::SignExtendData { .. } => {}
2282 Inst::XmmUnaryRmR {
2283 ref mut src,
2284 ref mut dst,
2285 ..
2286 }
2287 | Inst::XmmUnaryRmREvex {
2288 ref mut src,
2289 ref mut dst,
2290 ..
2291 }
2292 | Inst::UnaryRmR {
2293 ref mut src,
2294 ref mut dst,
2295 ..
2296 } => {
2297 src.map_uses(mapper);
2298 map_def(mapper, dst);
2299 }
2300 Inst::XmmRmRImm {
2301 ref op,
2302 ref mut src,
2303 ref mut dst,
2304 ..
2305 } => {
2306 if produces_const {
2307 src.map_as_def(mapper);
2308 map_def(mapper, dst);
2309 } else if *op == SseOpcode::Pextrb
2310 || *op == SseOpcode::Pextrw
2311 || *op == SseOpcode::Pextrd
2312 || *op == SseOpcode::Pshufd
2313 || *op == SseOpcode::Roundss
2314 || *op == SseOpcode::Roundsd
2315 || *op == SseOpcode::Roundps
2316 || *op == SseOpcode::Roundpd
2317 {
2318 src.map_uses(mapper);
2319 map_def(mapper, dst);
2320 } else {
2321 src.map_uses(mapper);
2322 map_mod(mapper, dst);
2323 }
2324 }
2325 Inst::XmmRmR {
2326 ref mut src,
2327 ref mut dst,
2328 ..
2329 } => {
2330 if produces_const {
2331 src.map_as_def(mapper);
2332 map_def(mapper, dst);
2333 } else {
2334 src.map_uses(mapper);
2335 map_mod(mapper, dst);
2336 }
2337 }
2338 Inst::XmmRmREvex {
2339 ref mut src1,
2340 ref mut src2,
2341 ref mut dst,
2342 ..
2343 } => {
2344 src1.map_uses(mapper);
2345 map_use(mapper, src2);
2346 map_def(mapper, dst);
2347 }
2348 Inst::XmmRmiReg {
2349 ref mut src,
2350 ref mut dst,
2351 ..
2352 } => {
2353 src.map_uses(mapper);
2354 map_mod(mapper, dst);
2355 }
2356 Inst::XmmUninitializedValue { ref mut dst, .. } => {
2357 map_def(mapper, dst);
2358 }
2359 Inst::XmmLoadConst { ref mut dst, .. } => {
2360 map_def(mapper, dst);
2361 }
2362 Inst::XmmMinMaxSeq {
2363 ref mut lhs,
2364 ref mut rhs_dst,
2365 ..
2366 } => {
2367 map_use(mapper, lhs);
2368 map_mod(mapper, rhs_dst);
2369 }
2370 Inst::XmmMovRM {
2371 ref mut src,
2372 ref mut dst,
2373 ..
2374 } => {
2375 map_use(mapper, src);
2376 dst.map_uses(mapper);
2377 }
2378 Inst::XmmCmpRmR {
2379 ref mut src,
2380 ref mut dst,
2381 ..
2382 } => {
2383 src.map_uses(mapper);
2384 map_use(mapper, dst);
2385 }
2386 Inst::Imm { ref mut dst, .. } => map_def(mapper, dst),
2387 Inst::MovRR {
2388 ref mut src,
2389 ref mut dst,
2390 ..
2391 }
2392 | Inst::XmmToGpr {
2393 ref mut src,
2394 ref mut dst,
2395 ..
2396 } => {
2397 map_use(mapper, src);
2398 map_def(mapper, dst);
2399 }
2400 Inst::GprToXmm {
2401 ref mut src,
2402 ref mut dst,
2403 ..
2404 } => {
2405 src.map_uses(mapper);
2406 map_def(mapper, dst);
2407 }
2408 Inst::CvtUint64ToFloatSeq {
2409 ref mut src,
2410 ref mut dst,
2411 ref mut tmp_gpr1,
2412 ref mut tmp_gpr2,
2413 ..
2414 } => {
2415 map_mod(mapper, src);
2416 map_def(mapper, dst);
2417 map_def(mapper, tmp_gpr1);
2418 map_def(mapper, tmp_gpr2);
2419 }
2420 Inst::CvtFloatToSintSeq {
2421 ref mut src,
2422 ref mut dst,
2423 ref mut tmp_xmm,
2424 ref mut tmp_gpr,
2425 ..
2426 }
2427 | Inst::CvtFloatToUintSeq {
2428 ref mut src,
2429 ref mut dst,
2430 ref mut tmp_gpr,
2431 ref mut tmp_xmm,
2432 ..
2433 } => {
2434 map_mod(mapper, src);
2435 map_def(mapper, dst);
2436 map_def(mapper, tmp_gpr);
2437 map_def(mapper, tmp_xmm);
2438 }
2439 Inst::MovzxRmR {
2440 ref mut src,
2441 ref mut dst,
2442 ..
2443 } => {
2444 src.map_uses(mapper);
2445 map_def(mapper, dst);
2446 }
2447 Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => {
2448 src.map_uses(mapper);
2449 map_def(mapper, dst);
2450 }
2451 Inst::MovsxRmR {
2452 ref mut src,
2453 ref mut dst,
2454 ..
2455 } => {
2456 src.map_uses(mapper);
2457 map_def(mapper, dst);
2458 }
2459 Inst::MovRM {
2460 ref mut src,
2461 ref mut dst,
2462 ..
2463 } => {
2464 map_use(mapper, src);
2465 dst.map_uses(mapper);
2466 }
2467 Inst::ShiftR { ref mut dst, .. } => {
2468 map_mod(mapper, dst);
2469 }
2470 Inst::CmpRmiR {
2471 ref mut src,
2472 ref mut dst,
2473 ..
2474 } => {
2475 src.map_uses(mapper);
2476 map_use(mapper, dst);
2477 }
2478 Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst),
2479 Inst::Cmove {
2480 ref mut src,
2481 ref mut dst,
2482 ..
2483 }
2484 | Inst::XmmCmove {
2485 ref mut src,
2486 ref mut dst,
2487 ..
2488 } => {
2489 src.map_uses(mapper);
2490 map_mod(mapper, dst)
2491 }
2492 Inst::Push64 { ref mut src } => src.map_uses(mapper),
2493 Inst::Pop64 { ref mut dst } => {
2494 map_def(mapper, dst);
2495 }
2496
2497 Inst::CallKnown {
2498 ref mut uses,
2499 ref mut defs,
2500 ..
2501 } => {
2502 for r in uses.iter_mut() {
2503 map_use(mapper, r);
2504 }
2505 for r in defs.iter_mut() {
2506 map_def(mapper, r);
2507 }
2508 }
2509
2510 Inst::CallUnknown {
2511 ref mut uses,
2512 ref mut defs,
2513 ref mut dest,
2514 ..
2515 } => {
2516 for r in uses.iter_mut() {
2517 map_use(mapper, r);
2518 }
2519 for r in defs.iter_mut() {
2520 map_def(mapper, r);
2521 }
2522 dest.map_uses(mapper);
2523 }
2524
2525 Inst::JmpTableSeq {
2526 ref mut idx,
2527 ref mut tmp1,
2528 ref mut tmp2,
2529 ..
2530 } => {
2531 map_use(mapper, idx);
2532 map_def(mapper, tmp1);
2533 map_def(mapper, tmp2);
2534 }
2535
2536 Inst::JmpUnknown { ref mut target } => target.map_uses(mapper),
2537
2538 Inst::LoadExtName { ref mut dst, .. } => map_def(mapper, dst),
2539
2540 Inst::LockCmpxchg {
2541 ref mut src,
2542 ref mut dst,
2543 ..
2544 } => {
2545 map_use(mapper, src);
2546 dst.map_uses(mapper);
2547 }
2548
2549 Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg),
2550
2551 Inst::Ret
2552 | Inst::EpiloguePlaceholder
2553 | Inst::JmpKnown { .. }
2554 | Inst::JmpCond { .. }
2555 | Inst::JmpIf { .. }
2556 | Inst::Nop { .. }
2557 | Inst::TrapIf { .. }
2558 | Inst::VirtualSPOffsetAdj { .. }
2559 | Inst::Ud2 { .. }
2560 | Inst::Hlt
2561 | Inst::AtomicRmwSeq { .. }
2562 | Inst::ElfTlsGetAddr { .. }
2563 | Inst::MachOTlsGetAddr { .. }
2564 | Inst::Fence { .. }
2565 | Inst::Unwind { .. } => {
2566 // Instruction doesn't explicitly mention any regs, so it can't have any virtual
2567 // regs that we'd need to remap. Hence no action required.
2568 }
2569 }
2570 }
2571
2572 //=============================================================================
2573 // Instructions: misc functions and external interface
2574
2575 impl MachInst for Inst {
get_regs(&self, collector: &mut RegUsageCollector)2576 fn get_regs(&self, collector: &mut RegUsageCollector) {
2577 x64_get_regs(&self, collector)
2578 }
2579
map_regs<RUM: RegUsageMapper>(&mut self, mapper: &RUM)2580 fn map_regs<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
2581 x64_map_regs(self, mapper);
2582 }
2583
is_move(&self) -> Option<(Writable<Reg>, Reg)>2584 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
2585 match self {
2586 // Note (carefully!) that a 32-bit mov *isn't* a no-op since it zeroes
2587 // out the upper 32 bits of the destination. For example, we could
2588 // conceivably use `movl %reg, %reg` to zero out the top 32 bits of
2589 // %reg.
2590 Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
2591 Some((*dst, *src))
2592 }
2593 // Note as well that MOVS[S|D] when used in the `XmmUnaryRmR` context are pure moves of
2594 // scalar floating-point values (and annotate `dst` as `def`s to the register allocator)
2595 // whereas the same operation in a packed context, e.g. `XMM_RM_R`, is used to merge a
2596 // value into the lowest lane of a vector (not a move).
2597 Self::XmmUnaryRmR { op, src, dst, .. }
2598 if *op == SseOpcode::Movss
2599 || *op == SseOpcode::Movsd
2600 || *op == SseOpcode::Movaps
2601 || *op == SseOpcode::Movapd
2602 || *op == SseOpcode::Movups
2603 || *op == SseOpcode::Movupd
2604 || *op == SseOpcode::Movdqa
2605 || *op == SseOpcode::Movdqu =>
2606 {
2607 if let RegMem::Reg { reg } = src {
2608 Some((*dst, *reg))
2609 } else {
2610 None
2611 }
2612 }
2613 _ => None,
2614 }
2615 }
2616
is_epilogue_placeholder(&self) -> bool2617 fn is_epilogue_placeholder(&self) -> bool {
2618 if let Self::EpiloguePlaceholder = self {
2619 true
2620 } else {
2621 false
2622 }
2623 }
2624
is_term<'a>(&'a self) -> MachTerminator<'a>2625 fn is_term<'a>(&'a self) -> MachTerminator<'a> {
2626 match self {
2627 // Interesting cases.
2628 &Self::Ret | &Self::EpiloguePlaceholder => MachTerminator::Ret,
2629 &Self::JmpKnown { dst } => MachTerminator::Uncond(dst),
2630 &Self::JmpCond {
2631 taken, not_taken, ..
2632 } => MachTerminator::Cond(taken, not_taken),
2633 &Self::JmpTableSeq {
2634 ref targets_for_term,
2635 ..
2636 } => MachTerminator::Indirect(&targets_for_term[..]),
2637 // All other cases are boring.
2638 _ => MachTerminator::None,
2639 }
2640 }
2641
stack_op_info(&self) -> Option<MachInstStackOpInfo>2642 fn stack_op_info(&self) -> Option<MachInstStackOpInfo> {
2643 match self {
2644 Self::VirtualSPOffsetAdj { offset } => Some(MachInstStackOpInfo::NomSPAdj(*offset)),
2645 Self::MovRM {
2646 size: OperandSize::Size8,
2647 src,
2648 dst: SyntheticAmode::NominalSPOffset { simm32 },
2649 } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
2650 Self::Mov64MR {
2651 src: SyntheticAmode::NominalSPOffset { simm32 },
2652 dst,
2653 } => Some(MachInstStackOpInfo::LoadNomSPOff(
2654 dst.to_reg(),
2655 *simm32 as i64,
2656 )),
2657 _ => None,
2658 }
2659 }
2660
gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst2661 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
2662 let rc_dst = dst_reg.to_reg().get_class();
2663 let rc_src = src_reg.get_class();
2664 // If this isn't true, we have gone way off the rails.
2665 debug_assert!(rc_dst == rc_src);
2666 match rc_dst {
2667 RegClass::I64 => Inst::mov_r_r(OperandSize::Size64, src_reg, dst_reg),
2668 RegClass::V128 => {
2669 // The Intel optimization manual, in "3.5.1.13 Zero-Latency MOV Instructions",
2670 // doesn't include MOVSS/MOVSD as instructions with zero-latency. Use movaps for
2671 // those, which may write more lanes that we need, but are specified to have
2672 // zero-latency.
2673 let opcode = match ty {
2674 types::F32 | types::F64 | types::F32X4 => SseOpcode::Movaps,
2675 types::F64X2 => SseOpcode::Movapd,
2676 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqa,
2677 _ => unimplemented!("unable to move type: {}", ty),
2678 };
2679 Inst::xmm_unary_rm_r(opcode, RegMem::reg(src_reg), dst_reg)
2680 }
2681 _ => panic!("gen_move(x64): unhandled regclass {:?}", rc_dst),
2682 }
2683 }
2684
gen_nop(preferred_size: usize) -> Inst2685 fn gen_nop(preferred_size: usize) -> Inst {
2686 Inst::nop(std::cmp::min(preferred_size, 15) as u8)
2687 }
2688
maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst>2689 fn maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst> {
2690 None
2691 }
2692
rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])>2693 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
2694 match ty {
2695 types::I8 => Ok((&[RegClass::I64], &[types::I8])),
2696 types::I16 => Ok((&[RegClass::I64], &[types::I16])),
2697 types::I32 => Ok((&[RegClass::I64], &[types::I32])),
2698 types::I64 => Ok((&[RegClass::I64], &[types::I64])),
2699 types::B1 => Ok((&[RegClass::I64], &[types::B1])),
2700 types::B8 => Ok((&[RegClass::I64], &[types::B8])),
2701 types::B16 => Ok((&[RegClass::I64], &[types::B16])),
2702 types::B32 => Ok((&[RegClass::I64], &[types::B32])),
2703 types::B64 => Ok((&[RegClass::I64], &[types::B64])),
2704 types::R32 => panic!("32-bit reftype pointer should never be seen on x86-64"),
2705 types::R64 => Ok((&[RegClass::I64], &[types::R64])),
2706 types::F32 => Ok((&[RegClass::V128], &[types::F32])),
2707 types::F64 => Ok((&[RegClass::V128], &[types::F64])),
2708 types::I128 => Ok((&[RegClass::I64, RegClass::I64], &[types::I64, types::I64])),
2709 types::B128 => Ok((&[RegClass::I64, RegClass::I64], &[types::B64, types::B64])),
2710 _ if ty.is_vector() => {
2711 assert!(ty.bits() <= 128);
2712 Ok((&[RegClass::V128], &[types::I8X16]))
2713 }
2714 types::IFLAGS | types::FFLAGS => Ok((&[RegClass::I64], &[types::I64])),
2715 _ => Err(CodegenError::Unsupported(format!(
2716 "Unexpected SSA-value type: {}",
2717 ty
2718 ))),
2719 }
2720 }
2721
gen_jump(label: MachLabel) -> Inst2722 fn gen_jump(label: MachLabel) -> Inst {
2723 Inst::jmp_known(label)
2724 }
2725
gen_constant<F: FnMut(Type) -> Writable<Reg>>( to_regs: ValueRegs<Writable<Reg>>, value: u128, ty: Type, mut alloc_tmp: F, ) -> SmallVec<[Self; 4]>2726 fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
2727 to_regs: ValueRegs<Writable<Reg>>,
2728 value: u128,
2729 ty: Type,
2730 mut alloc_tmp: F,
2731 ) -> SmallVec<[Self; 4]> {
2732 let mut ret = SmallVec::new();
2733 if ty == types::I128 {
2734 ret.push(Inst::imm(
2735 OperandSize::Size64,
2736 value as u64,
2737 to_regs.regs()[0],
2738 ));
2739 ret.push(Inst::imm(
2740 OperandSize::Size64,
2741 (value >> 64) as u64,
2742 to_regs.regs()[1],
2743 ));
2744 } else {
2745 let to_reg = to_regs
2746 .only_reg()
2747 .expect("multi-reg values not supported on x64");
2748 if ty == types::F32 {
2749 if value == 0 {
2750 ret.push(Inst::xmm_rm_r(
2751 SseOpcode::Xorps,
2752 RegMem::reg(to_reg.to_reg()),
2753 to_reg,
2754 ));
2755 } else {
2756 let tmp = alloc_tmp(types::I32);
2757 ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp));
2758
2759 ret.push(Inst::gpr_to_xmm(
2760 SseOpcode::Movd,
2761 RegMem::reg(tmp.to_reg()),
2762 OperandSize::Size32,
2763 to_reg,
2764 ));
2765 }
2766 } else if ty == types::F64 {
2767 if value == 0 {
2768 ret.push(Inst::xmm_rm_r(
2769 SseOpcode::Xorpd,
2770 RegMem::reg(to_reg.to_reg()),
2771 to_reg,
2772 ));
2773 } else {
2774 let tmp = alloc_tmp(types::I64);
2775 ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp));
2776
2777 ret.push(Inst::gpr_to_xmm(
2778 SseOpcode::Movq,
2779 RegMem::reg(tmp.to_reg()),
2780 OperandSize::Size64,
2781 to_reg,
2782 ));
2783 }
2784 } else {
2785 // Must be an integer type.
2786 debug_assert!(
2787 ty == types::B1
2788 || ty == types::I8
2789 || ty == types::B8
2790 || ty == types::I16
2791 || ty == types::B16
2792 || ty == types::I32
2793 || ty == types::B32
2794 || ty == types::I64
2795 || ty == types::B64
2796 || ty == types::R32
2797 || ty == types::R64
2798 );
2799 // Immediates must be 32 or 64 bits.
2800 // Smaller types are widened.
2801 let size = match OperandSize::from_ty(ty) {
2802 OperandSize::Size64 => OperandSize::Size64,
2803 _ => OperandSize::Size32,
2804 };
2805 if value == 0 {
2806 ret.push(Inst::alu_rmi_r(
2807 size,
2808 AluRmiROpcode::Xor,
2809 RegMemImm::reg(to_reg.to_reg()),
2810 to_reg,
2811 ));
2812 } else {
2813 let value = value as u64;
2814 ret.push(Inst::imm(size, value.into(), to_reg));
2815 }
2816 }
2817 }
2818 ret
2819 }
2820
reg_universe(flags: &Flags) -> RealRegUniverse2821 fn reg_universe(flags: &Flags) -> RealRegUniverse {
2822 create_reg_universe_systemv(flags)
2823 }
2824
worst_case_size() -> CodeOffset2825 fn worst_case_size() -> CodeOffset {
2826 15
2827 }
2828
ref_type_regclass(_: &settings::Flags) -> RegClass2829 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
2830 RegClass::I64
2831 }
2832
gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self2833 fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self {
2834 Inst::ValueLabelMarker { label, reg }
2835 }
2836
defines_value_label(&self) -> Option<(ValueLabel, Reg)>2837 fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
2838 match self {
2839 Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)),
2840 _ => None,
2841 }
2842 }
2843
2844 type LabelUse = LabelUse;
2845 }
2846
2847 /// State carried between emissions of a sequence of instructions.
2848 #[derive(Default, Clone, Debug)]
2849 pub struct EmitState {
2850 /// Addend to convert nominal-SP offsets to real-SP offsets at the current
2851 /// program point.
2852 pub(crate) virtual_sp_offset: i64,
2853 /// Offset of FP from nominal-SP.
2854 pub(crate) nominal_sp_to_fp: i64,
2855 /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
2856 stack_map: Option<StackMap>,
2857 /// Current source location.
2858 cur_srcloc: SourceLoc,
2859 }
2860
2861 /// Constant state used during emissions of a sequence of instructions.
2862 pub struct EmitInfo {
2863 flags: settings::Flags,
2864 isa_flags: x64_settings::Flags,
2865 }
2866
2867 impl EmitInfo {
new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self2868 pub(crate) fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
2869 Self { flags, isa_flags }
2870 }
2871 }
2872
2873 impl MachInstEmitInfo for EmitInfo {
flags(&self) -> &Flags2874 fn flags(&self) -> &Flags {
2875 &self.flags
2876 }
2877 }
2878
2879 impl MachInstEmit for Inst {
2880 type State = EmitState;
2881 type Info = EmitInfo;
2882
emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State)2883 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
2884 emit::emit(self, sink, info, state);
2885 }
2886
pretty_print(&self, mb_rru: Option<&RealRegUniverse>, _: &mut Self::State) -> String2887 fn pretty_print(&self, mb_rru: Option<&RealRegUniverse>, _: &mut Self::State) -> String {
2888 self.show_rru(mb_rru)
2889 }
2890 }
2891
2892 impl MachInstEmitState<Inst> for EmitState {
new(abi: &dyn ABICallee<I = Inst>) -> Self2893 fn new(abi: &dyn ABICallee<I = Inst>) -> Self {
2894 EmitState {
2895 virtual_sp_offset: 0,
2896 nominal_sp_to_fp: abi.frame_size() as i64,
2897 stack_map: None,
2898 cur_srcloc: SourceLoc::default(),
2899 }
2900 }
2901
pre_safepoint(&mut self, stack_map: StackMap)2902 fn pre_safepoint(&mut self, stack_map: StackMap) {
2903 self.stack_map = Some(stack_map);
2904 }
2905
pre_sourceloc(&mut self, srcloc: SourceLoc)2906 fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
2907 self.cur_srcloc = srcloc;
2908 }
2909 }
2910
2911 impl EmitState {
take_stack_map(&mut self) -> Option<StackMap>2912 fn take_stack_map(&mut self) -> Option<StackMap> {
2913 self.stack_map.take()
2914 }
2915
clear_post_insn(&mut self)2916 fn clear_post_insn(&mut self) {
2917 self.stack_map = None;
2918 }
2919
cur_srcloc(&self) -> SourceLoc2920 pub(crate) fn cur_srcloc(&self) -> SourceLoc {
2921 self.cur_srcloc
2922 }
2923 }
2924
2925 /// A label-use (internal relocation) in generated code.
2926 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2927 pub enum LabelUse {
2928 /// A 32-bit offset from location of relocation itself, added to the existing value at that
2929 /// location. Used for control flow instructions which consider an offset from the start of the
2930 /// next instruction (so the size of the payload -- 4 bytes -- is subtracted from the payload).
2931 JmpRel32,
2932
2933 /// A 32-bit offset from location of relocation itself, added to the existing value at that
2934 /// location.
2935 PCRel32,
2936 }
2937
2938 impl MachInstLabelUse for LabelUse {
2939 const ALIGN: CodeOffset = 1;
2940
max_pos_range(self) -> CodeOffset2941 fn max_pos_range(self) -> CodeOffset {
2942 match self {
2943 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
2944 }
2945 }
2946
max_neg_range(self) -> CodeOffset2947 fn max_neg_range(self) -> CodeOffset {
2948 match self {
2949 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
2950 }
2951 }
2952
patch_size(self) -> CodeOffset2953 fn patch_size(self) -> CodeOffset {
2954 match self {
2955 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
2956 }
2957 }
2958
patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset)2959 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2960 let pc_rel = (label_offset as i64) - (use_offset as i64);
2961 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2962 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2963 let pc_rel = pc_rel as u32;
2964 match self {
2965 LabelUse::JmpRel32 => {
2966 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2967 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
2968 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2969 }
2970 LabelUse::PCRel32 => {
2971 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2972 let value = pc_rel.wrapping_add(addend);
2973 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2974 }
2975 }
2976 }
2977
supports_veneer(self) -> bool2978 fn supports_veneer(self) -> bool {
2979 match self {
2980 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
2981 }
2982 }
2983
veneer_size(self) -> CodeOffset2984 fn veneer_size(self) -> CodeOffset {
2985 match self {
2986 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
2987 }
2988 }
2989
generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse)2990 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
2991 match self {
2992 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
2993 panic!("Veneer not supported for JumpRel32 label-use.");
2994 }
2995 }
2996 }
2997 }
2998