1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/Casting.h"
28 #include "llvm/Support/ErrorHandling.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "mccodeemitter"
36 
37 namespace {
38 
39 enum PrefixKind { None, REX, REX2, XOP, VEX2, VEX3, EVEX };
40 
41 static void emitByte(uint8_t C, SmallVectorImpl<char> &CB) { CB.push_back(C); }
42 
43 class X86OpcodePrefixHelper {
44   // REX (1 byte)
45   // +-----+ +------+
46   // | 40H | | WRXB |
47   // +-----+ +------+
48 
49   // REX2 (2 bytes)
50   // +-----+ +-------------------+
51   // | D5H | | M | R'X'B' | WRXB |
52   // +-----+ +-------------------+
53 
54   // XOP (3-byte)
55   // +-----+ +--------------+ +-------------------+
56   // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
57   // +-----+ +--------------+ +-------------------+
58 
59   // VEX2 (2 bytes)
60   // +-----+ +-------------------+
61   // | C5h | | R | vvvv | L | pp |
62   // +-----+ +-------------------+
63 
64   // VEX3 (3 bytes)
65   // +-----+ +--------------+ +-------------------+
66   // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
67   // +-----+ +--------------+ +-------------------+
68 
69   // VEX_R: opcode externsion equivalent to REX.R in
70   // 1's complement (inverted) form
71   //
72   //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
73   //  0: Same as REX_R=1 (64 bit mode only)
74 
75   // VEX_X: equivalent to REX.X, only used when a
76   // register is used for index in SIB Byte.
77   //
78   //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
79   //  0: Same as REX.X=1 (64-bit mode only)
80 
81   // VEX_B:
82   //  1: Same as REX_B=0 (ignored in 32-bit mode)
83   //  0: Same as REX_B=1 (64 bit mode only)
84 
85   // VEX_W: opcode specific (use like REX.W, or used for
86   // opcode extension, or ignored, depending on the opcode byte)
87 
88   // VEX_5M (VEX m-mmmmm field):
89   //
90   //  0b00000: Reserved for future use
91   //  0b00001: implied 0F leading opcode
92   //  0b00010: implied 0F 38 leading opcode bytes
93   //  0b00011: implied 0F 3A leading opcode bytes
94   //  0b00100: Reserved for future use
95   //  0b00101: VEX MAP5
96   //  0b00110: VEX MAP6
97   //  0b00111: VEX MAP7
98   //  0b00111-0b11111: Reserved for future use
99   //  0b01000: XOP map select - 08h instructions with imm byte
100   //  0b01001: XOP map select - 09h instructions with no imm byte
101   //  0b01010: XOP map select - 0Ah instructions with imm dword
102 
103   // VEX_4V (VEX vvvv field): a register specifier
104   // (in 1's complement form) or 1111 if unused.
105 
106   // VEX_PP: opcode extension providing equivalent
107   // functionality of a SIMD prefix
108   //  0b00: None
109   //  0b01: 66
110   //  0b10: F3
111   //  0b11: F2
112 
113   // EVEX (4 bytes)
114   // +-----+ +---------------+ +--------------------+ +------------------------+
115   // | 62h | | RXBR' | B'mmm | | W | vvvv | X' | pp | | z | L'L | b | v' | aaa |
116   // +-----+ +---------------+ +--------------------+ +------------------------+
117 
118   // EVEX_L2/VEX_L (Vector Length):
119   // L2 L
120   //  0 0: scalar or 128-bit vector
121   //  0 1: 256-bit vector
122   //  1 0: 512-bit vector
123 
124   // 32-Register Support in 64-bit Mode Using EVEX with Embedded REX/REX2 Bits:
125   //
126   // +----------+---------+--------+-----------+---------+--------------+
127   // |          |    4    |    3   |   [2:0]   | Type    | Common Usage |
128   // +----------+---------+--------+-----------+---------+--------------+
129   // | REG      | EVEX_R' | EVEX_R | modrm.reg | GPR, VR | Dest or Src  |
130   // | VVVV     | EVEX_v' |       EVEX.vvvv    | GPR, VR | Dest or Src  |
131   // | RM (VR)  | EVEX_X  | EVEX_B | modrm.r/m | VR      | Dest or Src  |
132   // | RM (GPR) | EVEX_B' | EVEX_B | modrm.r/m | GPR     | Dest or Src  |
133   // | BASE     | EVEX_B' | EVEX_B | modrm.r/m | GPR     | MA           |
134   // | INDEX    | EVEX_X' | EVEX_X | sib.index | GPR     | MA           |
135   // | VIDX     | EVEX_v' | EVEX_X | sib.index | VR      | VSIB MA      |
136   // +----------+---------+--------+-----------+---------+--------------+
137   //
138   // * GPR  - General-purpose register
139   // * VR   - Vector register
140   // * VIDX - Vector index
141   // * VSIB - Vector SIB
142   // * MA   - Memory addressing
143 
144 private:
145   unsigned W : 1;
146   unsigned R : 1;
147   unsigned X : 1;
148   unsigned B : 1;
149   unsigned M : 1;
150   unsigned R2 : 1;
151   unsigned X2 : 1;
152   unsigned B2 : 1;
153   unsigned VEX_4V : 4;
154   unsigned VEX_L : 1;
155   unsigned VEX_PP : 2;
156   unsigned VEX_5M : 5;
157   unsigned EVEX_z : 1;
158   unsigned EVEX_L2 : 1;
159   unsigned EVEX_b : 1;
160   unsigned EVEX_V2 : 1;
161   unsigned EVEX_aaa : 3;
162   PrefixKind Kind = None;
163   const MCRegisterInfo &MRI;
164 
165   unsigned getRegEncoding(const MCInst &MI, unsigned OpNum) const {
166     return MRI.getEncodingValue(MI.getOperand(OpNum).getReg());
167   }
168 
169   void setR(unsigned Encoding) { R = Encoding >> 3 & 1; }
170   void setR2(unsigned Encoding) {
171     R2 = Encoding >> 4 & 1;
172     assert((!R2 || (Kind <= REX2 || Kind == EVEX)) && "invalid setting");
173   }
174   void setX(unsigned Encoding) { X = Encoding >> 3 & 1; }
175   void setX2(unsigned Encoding) {
176     assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");
177     X2 = Encoding >> 4 & 1;
178   }
179   void setB(unsigned Encoding) { B = Encoding >> 3 & 1; }
180   void setB2(unsigned Encoding) {
181     assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");
182     B2 = Encoding >> 4 & 1;
183   }
184   void set4V(unsigned Encoding) { VEX_4V = Encoding & 0xf; }
185   void setV2(unsigned Encoding) { EVEX_V2 = Encoding >> 4 & 1; }
186 
187 public:
188   void setW(bool V) { W = V; }
189   void setR(const MCInst &MI, unsigned OpNum) {
190     setR(getRegEncoding(MI, OpNum));
191   }
192   void setX(const MCInst &MI, unsigned OpNum, unsigned Shift = 3) {
193     unsigned Reg = MI.getOperand(OpNum).getReg();
194     // X is used to extend vector register only when shift is not 3.
195     if (Shift != 3 && X86II::isApxExtendedReg(Reg))
196       return;
197     unsigned Encoding = MRI.getEncodingValue(Reg);
198     X = Encoding >> Shift & 1;
199   }
200   void setB(const MCInst &MI, unsigned OpNum) {
201     B = getRegEncoding(MI, OpNum) >> 3 & 1;
202   }
203   void set4V(const MCInst &MI, unsigned OpNum) {
204     set4V(getRegEncoding(MI, OpNum));
205   }
206   void setL(bool V) { VEX_L = V; }
207   void setPP(unsigned V) { VEX_PP = V; }
208   void set5M(unsigned V) { VEX_5M = V; }
209   void setR2(const MCInst &MI, unsigned OpNum) {
210     setR2(getRegEncoding(MI, OpNum));
211   }
212   void setRR2(const MCInst &MI, unsigned OpNum) {
213     unsigned Encoding = getRegEncoding(MI, OpNum);
214     setR(Encoding);
215     setR2(Encoding);
216   }
217   void setM(bool V) { M = V; }
218   void setXX2(const MCInst &MI, unsigned OpNum) {
219     unsigned Reg = MI.getOperand(OpNum).getReg();
220     unsigned Encoding = MRI.getEncodingValue(Reg);
221     setX(Encoding);
222     // Index can be a vector register while X2 is used to extend GPR only.
223     if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))
224       setX2(Encoding);
225   }
226   void setBB2(const MCInst &MI, unsigned OpNum) {
227     unsigned Reg = MI.getOperand(OpNum).getReg();
228     unsigned Encoding = MRI.getEncodingValue(Reg);
229     setB(Encoding);
230     // Base can be a vector register while B2 is used to extend GPR only
231     if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))
232       setB2(Encoding);
233   }
234   void setZ(bool V) { EVEX_z = V; }
235   void setL2(bool V) { EVEX_L2 = V; }
236   void setEVEX_b(bool V) { EVEX_b = V; }
237   void setV2(const MCInst &MI, unsigned OpNum, bool HasVEX_4V) {
238     // Only needed with VSIB which don't use VVVV.
239     if (HasVEX_4V)
240       return;
241     unsigned Reg = MI.getOperand(OpNum).getReg();
242     if (X86II::isApxExtendedReg(Reg))
243       return;
244     setV2(MRI.getEncodingValue(Reg));
245   }
246   void set4VV2(const MCInst &MI, unsigned OpNum) {
247     unsigned Encoding = getRegEncoding(MI, OpNum);
248     set4V(Encoding);
249     setV2(Encoding);
250   }
251   void setAAA(const MCInst &MI, unsigned OpNum) {
252     EVEX_aaa = getRegEncoding(MI, OpNum);
253   }
254   void setNF(bool V) { EVEX_aaa |= V << 2; }
255 
256   X86OpcodePrefixHelper(const MCRegisterInfo &MRI)
257       : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0),
258         VEX_PP(0), VEX_5M(0), EVEX_z(0), EVEX_L2(0), EVEX_b(0), EVEX_V2(0),
259         EVEX_aaa(0), MRI(MRI) {}
260 
261   void setLowerBound(PrefixKind K) { Kind = K; }
262 
263   PrefixKind determineOptimalKind() {
264     switch (Kind) {
265     case None:
266       // Not M bit here by intention b/c
267       // 1. No guarantee that REX2 is supported by arch w/o explict EGPR
268       // 2. REX2 is longer than 0FH
269       Kind = (R2 | X2 | B2) ? REX2 : (W | R | X | B) ? REX : None;
270       break;
271     case REX:
272       Kind = (R2 | X2 | B2) ? REX2 : REX;
273       break;
274     case REX2:
275     case XOP:
276     case VEX3:
277     case EVEX:
278       break;
279     case VEX2:
280       Kind = (W | X | B | (VEX_5M != 1)) ? VEX3 : VEX2;
281       break;
282     }
283     return Kind;
284   }
285 
286   void emit(SmallVectorImpl<char> &CB) const {
287     uint8_t FirstPayload =
288         ((~R) & 0x1) << 7 | ((~X) & 0x1) << 6 | ((~B) & 0x1) << 5;
289     uint8_t LastPayload = ((~VEX_4V) & 0xf) << 3 | VEX_L << 2 | VEX_PP;
290     switch (Kind) {
291     case None:
292       return;
293     case REX:
294       emitByte(0x40 | W << 3 | R << 2 | X << 1 | B, CB);
295       return;
296     case REX2:
297       emitByte(0xD5, CB);
298       emitByte(M << 7 | R2 << 6 | X2 << 5 | B2 << 4 | W << 3 | R << 2 | X << 1 |
299                    B,
300                CB);
301       return;
302     case VEX2:
303       emitByte(0xC5, CB);
304       emitByte(((~R) & 1) << 7 | LastPayload, CB);
305       return;
306     case VEX3:
307     case XOP:
308       emitByte(Kind == VEX3 ? 0xC4 : 0x8F, CB);
309       emitByte(FirstPayload | VEX_5M, CB);
310       emitByte(W << 7 | LastPayload, CB);
311       return;
312     case EVEX:
313       assert(VEX_5M && !(VEX_5M & 0x8) && "invalid mmm fields for EVEX!");
314       emitByte(0x62, CB);
315       emitByte(FirstPayload | ((~R2) & 0x1) << 4 | B2 << 3 | VEX_5M, CB);
316       emitByte(W << 7 | ((~VEX_4V) & 0xf) << 3 | ((~X2) & 0x1) << 2 | VEX_PP,
317                CB);
318       emitByte(EVEX_z << 7 | EVEX_L2 << 6 | VEX_L << 5 | EVEX_b << 4 |
319                    ((~EVEX_V2) & 0x1) << 3 | EVEX_aaa,
320                CB);
321       return;
322     }
323   }
324 };
325 
326 class X86MCCodeEmitter : public MCCodeEmitter {
327   const MCInstrInfo &MCII;
328   MCContext &Ctx;
329 
330 public:
331   X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
332       : MCII(mcii), Ctx(ctx) {}
333   X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
334   X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
335   ~X86MCCodeEmitter() override = default;
336 
337   void emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,
338                   const MCSubtargetInfo &STI) const override;
339 
340   void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,
341                          SmallVectorImpl<MCFixup> &Fixups,
342                          const MCSubtargetInfo &STI) const override;
343 
344 private:
345   unsigned getX86RegNum(const MCOperand &MO) const;
346 
347   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
348 
349   void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
350                      MCFixupKind FixupKind, uint64_t StartByte,
351                      SmallVectorImpl<char> &CB,
352                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
353 
354   void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
355                         SmallVectorImpl<char> &CB) const;
356 
357   void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
358                    SmallVectorImpl<char> &CB) const;
359 
360   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
361                         uint64_t TSFlags, PrefixKind Kind, uint64_t StartByte,
362                         SmallVectorImpl<char> &CB,
363                         SmallVectorImpl<MCFixup> &Fixups,
364                         const MCSubtargetInfo &STI,
365                         bool ForceSIB = false) const;
366 
367   PrefixKind emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
368                             const MCSubtargetInfo &STI,
369                             SmallVectorImpl<char> &CB) const;
370 
371   PrefixKind emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
372                                  const MCSubtargetInfo &STI,
373                                  SmallVectorImpl<char> &CB) const;
374 
375   void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
376                                  SmallVectorImpl<char> &CB) const;
377 
378   PrefixKind emitOpcodePrefix(int MemOperand, const MCInst &MI,
379                               const MCSubtargetInfo &STI,
380                               SmallVectorImpl<char> &CB) const;
381 
382   PrefixKind emitREXPrefix(int MemOperand, const MCInst &MI,
383                            const MCSubtargetInfo &STI,
384                            SmallVectorImpl<char> &CB) const;
385 };
386 
387 } // end anonymous namespace
388 
389 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
390   assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
391   return RM | (RegOpcode << 3) | (Mod << 6);
392 }
393 
394 static void emitConstant(uint64_t Val, unsigned Size,
395                          SmallVectorImpl<char> &CB) {
396   // Output the constant in little endian byte order.
397   for (unsigned i = 0; i != Size; ++i) {
398     emitByte(Val & 255, CB);
399     Val >>= 8;
400   }
401 }
402 
403 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
404 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
405 /// parameter of emitImmediate.
406 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
407   bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
408 
409   unsigned CD8_Scale =
410       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
411   CD8_Scale = CD8_Scale ? 1U << (CD8_Scale - 1) : 0U;
412   if (!HasEVEX || !CD8_Scale)
413     return isInt<8>(Value);
414 
415   assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
416   if (Value & (CD8_Scale - 1)) // Unaligned offset
417     return false;
418 
419   int CDisp8 = Value / static_cast<int>(CD8_Scale);
420   if (!isInt<8>(CDisp8))
421     return false;
422 
423   // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
424   ImmOffset = CDisp8 - Value;
425   return true;
426 }
427 
428 /// \returns the appropriate fixup kind to use for an immediate in an
429 /// instruction with the specified TSFlags.
430 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
431   unsigned Size = X86II::getSizeOfImm(TSFlags);
432   bool isPCRel = X86II::isImmPCRel(TSFlags);
433 
434   if (X86II::isImmSigned(TSFlags)) {
435     switch (Size) {
436     default:
437       llvm_unreachable("Unsupported signed fixup size!");
438     case 4:
439       return MCFixupKind(X86::reloc_signed_4byte);
440     }
441   }
442   return MCFixup::getKindForSize(Size, isPCRel);
443 }
444 
445 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
446 
447 /// Check if this expression starts with  _GLOBAL_OFFSET_TABLE_ and if it is
448 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
449 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
450 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
451 /// binary expression.
452 static GlobalOffsetTableExprKind
453 startsWithGlobalOffsetTable(const MCExpr *Expr) {
454   const MCExpr *RHS = nullptr;
455   if (Expr->getKind() == MCExpr::Binary) {
456     const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
457     Expr = BE->getLHS();
458     RHS = BE->getRHS();
459   }
460 
461   if (Expr->getKind() != MCExpr::SymbolRef)
462     return GOT_None;
463 
464   const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
465   const MCSymbol &S = Ref->getSymbol();
466   if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
467     return GOT_None;
468   if (RHS && RHS->getKind() == MCExpr::SymbolRef)
469     return GOT_SymDiff;
470   return GOT_Normal;
471 }
472 
473 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
474   if (Expr->getKind() == MCExpr::SymbolRef) {
475     const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
476     return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
477   }
478   return false;
479 }
480 
481 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
482   unsigned Opcode = MI.getOpcode();
483   const MCInstrDesc &Desc = MCII.get(Opcode);
484   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
485        Opcode != X86::JCC_4) ||
486       getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
487     return false;
488 
489   unsigned CurOp = X86II::getOperandBias(Desc);
490   const MCOperand &Op = MI.getOperand(CurOp);
491   if (!Op.isExpr())
492     return false;
493 
494   const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
495   return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
496 }
497 
498 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
499   return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
500 }
501 
502 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
503                                              unsigned OpNum) const {
504   return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
505 }
506 
507 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
508                                      unsigned Size, MCFixupKind FixupKind,
509                                      uint64_t StartByte,
510                                      SmallVectorImpl<char> &CB,
511                                      SmallVectorImpl<MCFixup> &Fixups,
512                                      int ImmOffset) const {
513   const MCExpr *Expr = nullptr;
514   if (DispOp.isImm()) {
515     // If this is a simple integer displacement that doesn't require a
516     // relocation, emit it now.
517     if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
518         FixupKind != FK_PCRel_4) {
519       emitConstant(DispOp.getImm() + ImmOffset, Size, CB);
520       return;
521     }
522     Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
523   } else {
524     Expr = DispOp.getExpr();
525   }
526 
527   // If we have an immoffset, add it to the expression.
528   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
529        FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
530     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
531     if (Kind != GOT_None) {
532       assert(ImmOffset == 0);
533 
534       if (Size == 8) {
535         FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
536       } else {
537         assert(Size == 4);
538         FixupKind = MCFixupKind(X86::reloc_global_offset_table);
539       }
540 
541       if (Kind == GOT_Normal)
542         ImmOffset = static_cast<int>(CB.size() - StartByte);
543     } else if (Expr->getKind() == MCExpr::SymbolRef) {
544       if (hasSecRelSymbolRef(Expr)) {
545         FixupKind = MCFixupKind(FK_SecRel_4);
546       }
547     } else if (Expr->getKind() == MCExpr::Binary) {
548       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
549       if (hasSecRelSymbolRef(Bin->getLHS()) ||
550           hasSecRelSymbolRef(Bin->getRHS())) {
551         FixupKind = MCFixupKind(FK_SecRel_4);
552       }
553     }
554   }
555 
556   // If the fixup is pc-relative, we need to bias the value to be relative to
557   // the start of the field, not the end of the field.
558   if (FixupKind == FK_PCRel_4 ||
559       FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
560       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
561       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
562       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
563       FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
564     ImmOffset -= 4;
565     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
566     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
567     // this needs to be a GOTPC32 relocation.
568     if (startsWithGlobalOffsetTable(Expr) != GOT_None)
569       FixupKind = MCFixupKind(X86::reloc_global_offset_table);
570   }
571   if (FixupKind == FK_PCRel_2)
572     ImmOffset -= 2;
573   if (FixupKind == FK_PCRel_1)
574     ImmOffset -= 1;
575 
576   if (ImmOffset)
577     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
578                                    Ctx);
579 
580   // Emit a symbolic constant as a fixup and 4 zeros.
581   Fixups.push_back(MCFixup::create(static_cast<uint32_t>(CB.size() - StartByte),
582                                    Expr, FixupKind, Loc));
583   emitConstant(0, Size, CB);
584 }
585 
586 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
587                                         unsigned RegOpcodeFld,
588                                         SmallVectorImpl<char> &CB) const {
589   emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), CB);
590 }
591 
592 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
593                                    SmallVectorImpl<char> &CB) const {
594   // SIB byte is in the same format as the modRMByte.
595   emitByte(modRMByte(SS, Index, Base), CB);
596 }
597 
598 void X86MCCodeEmitter::emitMemModRMByte(
599     const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags,
600     PrefixKind Kind, uint64_t StartByte, SmallVectorImpl<char> &CB,
601     SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI,
602     bool ForceSIB) const {
603   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
604   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
605   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
606   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
607   unsigned BaseReg = Base.getReg();
608 
609   // Handle %rip relative addressing.
610   if (BaseReg == X86::RIP ||
611       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
612     assert(STI.hasFeature(X86::Is64Bit) &&
613            "Rip-relative addressing requires 64-bit mode");
614     assert(IndexReg.getReg() == 0 && !ForceSIB &&
615            "Invalid rip-relative address");
616     emitByte(modRMByte(0, RegOpcodeField, 5), CB);
617 
618     unsigned Opcode = MI.getOpcode();
619     unsigned FixupKind = [&]() {
620       // Enable relaxed relocation only for a MCSymbolRefExpr.  We cannot use a
621       // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
622       if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))
623         return X86::reloc_riprel_4byte;
624 
625       // Certain loads for GOT references can be relocated against the symbol
626       // directly if the symbol ends up in the same linkage unit.
627       switch (Opcode) {
628       default:
629         return X86::reloc_riprel_4byte;
630       case X86::MOV64rm:
631         // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a
632         // special case because COFF and Mach-O don't support ELF's more
633         // flexible R_X86_64_REX_GOTPCRELX relaxation.
634         // TODO: Support new relocation for REX2.
635         assert(Kind == REX || Kind == REX2);
636         return X86::reloc_riprel_4byte_movq_load;
637       case X86::ADC32rm:
638       case X86::ADD32rm:
639       case X86::AND32rm:
640       case X86::CMP32rm:
641       case X86::MOV32rm:
642       case X86::OR32rm:
643       case X86::SBB32rm:
644       case X86::SUB32rm:
645       case X86::TEST32mr:
646       case X86::XOR32rm:
647       case X86::CALL64m:
648       case X86::JMP64m:
649       case X86::TAILJMPm64:
650       case X86::TEST64mr:
651       case X86::ADC64rm:
652       case X86::ADD64rm:
653       case X86::AND64rm:
654       case X86::CMP64rm:
655       case X86::OR64rm:
656       case X86::SBB64rm:
657       case X86::SUB64rm:
658       case X86::XOR64rm:
659         // We haven't support relocation for REX2 prefix, so temporarily use REX
660         // relocation.
661         // TODO: Support new relocation for REX2.
662         return (Kind == REX || Kind == REX2) ? X86::reloc_riprel_4byte_relax_rex
663                                              : X86::reloc_riprel_4byte_relax;
664       }
665     }();
666 
667     // rip-relative addressing is actually relative to the *next* instruction.
668     // Since an immediate can follow the mod/rm byte for an instruction, this
669     // means that we need to bias the displacement field of the instruction with
670     // the size of the immediate field. If we have this case, add it into the
671     // expression to emit.
672     // Note: rip-relative addressing using immediate displacement values should
673     // not be adjusted, assuming it was the user's intent.
674     int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
675                       ? X86II::getSizeOfImm(TSFlags)
676                       : 0;
677 
678     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
679                   Fixups, -ImmSize);
680     return;
681   }
682 
683   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
684 
685   // 16-bit addressing forms of the ModR/M byte have a different encoding for
686   // the R/M field and are far more limited in which registers can be used.
687   if (X86_MC::is16BitMemOperand(MI, Op, STI)) {
688     if (BaseReg) {
689       // For 32-bit addressing, the row and column values in Table 2-2 are
690       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
691       // some special cases. And getX86RegNum reflects that numbering.
692       // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
693       // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
694       // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
695       // while values 0-3 indicate the allowed combinations (base+index) of
696       // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
697       //
698       // R16Table[] is a lookup from the normal RegNo, to the row values from
699       // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
700       static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
701       unsigned RMfield = R16Table[BaseRegNo];
702 
703       assert(RMfield && "invalid 16-bit base register");
704 
705       if (IndexReg.getReg()) {
706         unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
707 
708         assert(IndexReg16 && "invalid 16-bit index register");
709         // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
710         assert(((IndexReg16 ^ RMfield) & 2) &&
711                "invalid 16-bit base/index register combination");
712         assert(Scale.getImm() == 1 &&
713                "invalid scale for 16-bit memory reference");
714 
715         // Allow base/index to appear in either order (although GAS doesn't).
716         if (IndexReg16 & 2)
717           RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
718         else
719           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
720       }
721 
722       if (Disp.isImm() && isInt<8>(Disp.getImm())) {
723         if (Disp.getImm() == 0 && RMfield != 6) {
724           // There is no displacement; just the register.
725           emitByte(modRMByte(0, RegOpcodeField, RMfield), CB);
726           return;
727         }
728         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
729         emitByte(modRMByte(1, RegOpcodeField, RMfield), CB);
730         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups);
731         return;
732       }
733       // This is the [REG]+disp16 case.
734       emitByte(modRMByte(2, RegOpcodeField, RMfield), CB);
735     } else {
736       assert(IndexReg.getReg() == 0 && "Unexpected index register!");
737       // There is no BaseReg; this is the plain [disp16] case.
738       emitByte(modRMByte(0, RegOpcodeField, 6), CB);
739     }
740 
741     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
742     emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, CB, Fixups);
743     return;
744   }
745 
746   // Check for presence of {disp8} or {disp32} pseudo prefixes.
747   bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
748   bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
749 
750   // We only allow no displacement if no pseudo prefix is present.
751   bool AllowNoDisp = !UseDisp8 && !UseDisp32;
752   // Disp8 is allowed unless the {disp32} prefix is present.
753   bool AllowDisp8 = !UseDisp32;
754 
755   // Determine whether a SIB byte is needed.
756   if ( // The SIB byte must be used if there is an index register or the
757        // encoding requires a SIB byte.
758       !ForceSIB && IndexReg.getReg() == 0 &&
759       // The SIB byte must be used if the base is ESP/RSP/R12/R20/R28, all of
760       // which encode to an R/M value of 4, which indicates that a SIB byte is
761       // present.
762       BaseRegNo != N86::ESP &&
763       // If there is no base register and we're in 64-bit mode, we need a SIB
764       // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
765       (!STI.hasFeature(X86::Is64Bit) || BaseReg != 0)) {
766 
767     if (BaseReg == 0) { // [disp32]     in X86-32 mode
768       emitByte(modRMByte(0, RegOpcodeField, 5), CB);
769       emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, CB, Fixups);
770       return;
771     }
772 
773     // If the base is not EBP/ESP/R12/R13/R20/R21/R28/R29 and there is no
774     // displacement, use simple indirect register encoding, this handles
775     // addresses like [EAX]. The encoding for [EBP], [R13], [R20], [R21], [R28]
776     // or [R29] with no displacement means [disp32] so we handle it by emitting
777     // a displacement of 0 later.
778     if (BaseRegNo != N86::EBP) {
779       if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
780         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);
781         return;
782       }
783 
784       // If the displacement is @tlscall, treat it as a zero.
785       if (Disp.isExpr()) {
786         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
787         if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
788           // This is exclusively used by call *a@tlscall(base). The relocation
789           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
790           Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
791           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);
792           return;
793         }
794       }
795     }
796 
797     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
798     // Including a compressed disp8 for EVEX instructions that support it.
799     // This also handles the 0 displacement for [EBP], [R13], [R21] or [R29]. We
800     // can't use disp8 if the {disp32} pseudo prefix is present.
801     if (Disp.isImm() && AllowDisp8) {
802       int ImmOffset = 0;
803       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
804         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CB);
805         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
806                       ImmOffset);
807         return;
808       }
809     }
810 
811     // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
812     // Displacement may be 0 for [EBP], [R13], [R21], [R29] case if {disp32}
813     // pseudo prefix prevented using disp8 above.
814     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), CB);
815     unsigned Opcode = MI.getOpcode();
816     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
817                                                 : X86::reloc_signed_4byte;
818     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
819                   Fixups);
820     return;
821   }
822 
823   // We need a SIB byte, so start by outputting the ModR/M byte first
824   assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
825          "Cannot use ESP as index reg!");
826 
827   bool ForceDisp32 = false;
828   bool ForceDisp8 = false;
829   int ImmOffset = 0;
830   if (BaseReg == 0) {
831     // If there is no base register, we emit the special case SIB byte with
832     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
833     BaseRegNo = 5;
834     emitByte(modRMByte(0, RegOpcodeField, 4), CB);
835     ForceDisp32 = true;
836   } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
837              // Base reg can't be EBP/RBP/R13/R21/R29 as that would end up with
838              // '5' as the base field, but that is the magic [*] nomenclature
839              // that indicates no base when mod=0. For these cases we'll emit a
840              // 0 displacement instead.
841              BaseRegNo != N86::EBP) {
842     // Emit no displacement ModR/M byte
843     emitByte(modRMByte(0, RegOpcodeField, 4), CB);
844   } else if (Disp.isImm() && AllowDisp8 &&
845              isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
846     // Displacement fits in a byte or matches an EVEX compressed disp8, use
847     // disp8 encoding. This also handles EBP/R13/R21/R29 base with 0
848     // displacement unless {disp32} pseudo prefix was used.
849     emitByte(modRMByte(1, RegOpcodeField, 4), CB);
850     ForceDisp8 = true;
851   } else {
852     // Otherwise, emit the normal disp32 encoding.
853     emitByte(modRMByte(2, RegOpcodeField, 4), CB);
854     ForceDisp32 = true;
855   }
856 
857   // Calculate what the SS field value should be...
858   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
859   unsigned SS = SSTable[Scale.getImm()];
860 
861   unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
862 
863   emitSIBByte(SS, IndexRegNo, BaseRegNo, CB);
864 
865   // Do we need to output a displacement?
866   if (ForceDisp8)
867     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
868                   ImmOffset);
869   else if (ForceDisp32)
870     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
871                   StartByte, CB, Fixups);
872 }
873 
874 /// Emit all instruction prefixes.
875 ///
876 /// \returns one of the REX, XOP, VEX2, VEX3, EVEX if any of them is used,
877 /// otherwise returns None.
878 PrefixKind X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
879                                             const MCSubtargetInfo &STI,
880                                             SmallVectorImpl<char> &CB) const {
881   uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
882   // Determine where the memory operand starts, if present.
883   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
884   // Emit segment override opcode prefix as needed.
885   if (MemoryOperand != -1) {
886     MemoryOperand += CurOp;
887     emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, CB);
888   }
889 
890   // Emit the repeat opcode prefix as needed.
891   unsigned Flags = MI.getFlags();
892   if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
893     emitByte(0xF3, CB);
894   if (Flags & X86::IP_HAS_REPEAT_NE)
895     emitByte(0xF2, CB);
896 
897   // Emit the address size opcode prefix as needed.
898   if (X86_MC::needsAddressSizeOverride(MI, STI, MemoryOperand, TSFlags) ||
899       Flags & X86::IP_HAS_AD_SIZE)
900     emitByte(0x67, CB);
901 
902   uint64_t Form = TSFlags & X86II::FormMask;
903   switch (Form) {
904   default:
905     break;
906   case X86II::RawFrmDstSrc: {
907     // Emit segment override opcode prefix as needed (not for %ds).
908     if (MI.getOperand(2).getReg() != X86::DS)
909       emitSegmentOverridePrefix(2, MI, CB);
910     CurOp += 3; // Consume operands.
911     break;
912   }
913   case X86II::RawFrmSrc: {
914     // Emit segment override opcode prefix as needed (not for %ds).
915     if (MI.getOperand(1).getReg() != X86::DS)
916       emitSegmentOverridePrefix(1, MI, CB);
917     CurOp += 2; // Consume operands.
918     break;
919   }
920   case X86II::RawFrmDst: {
921     ++CurOp; // Consume operand.
922     break;
923   }
924   case X86II::RawFrmMemOffs: {
925     // Emit segment override opcode prefix as needed.
926     emitSegmentOverridePrefix(1, MI, CB);
927     break;
928   }
929   }
930 
931   // REX prefix is optional, but if used must be immediately before the opcode
932   // Encoding type for this instruction.
933   return (TSFlags & X86II::EncodingMask)
934              ? emitVEXOpcodePrefix(MemoryOperand, MI, STI, CB)
935              : emitOpcodePrefix(MemoryOperand, MI, STI, CB);
936 }
937 
938 // AVX instructions are encoded using an encoding scheme that combines
939 // prefix bytes, opcode extension field, operand encoding fields, and vector
940 // length encoding capability into a new prefix, referred to as VEX.
941 
942 // The majority of the AVX-512 family of instructions (operating on
943 // 512/256/128-bit vector register operands) are encoded using a new prefix
944 // (called EVEX).
945 
946 // XOP is a revised subset of what was originally intended as SSE5. It was
947 // changed to be similar but not overlapping with AVX.
948 
949 /// Emit XOP, VEX2, VEX3 or EVEX prefix.
950 /// \returns the used prefix.
951 PrefixKind
952 X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
953                                       const MCSubtargetInfo &STI,
954                                       SmallVectorImpl<char> &CB) const {
955   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
956   uint64_t TSFlags = Desc.TSFlags;
957 
958   assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
959 
960 #ifndef NDEBUG
961   unsigned NumOps = MI.getNumOperands();
962   for (unsigned I = NumOps ? X86II::getOperandBias(Desc) : 0; I != NumOps;
963        ++I) {
964     const MCOperand &MO = MI.getOperand(I);
965     if (!MO.isReg())
966       continue;
967     unsigned Reg = MO.getReg();
968     if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
969       report_fatal_error(
970           "Cannot encode high byte register in VEX/EVEX-prefixed instruction");
971   }
972 #endif
973 
974   X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());
975   switch (TSFlags & X86II::EncodingMask) {
976   default:
977     break;
978   case X86II::XOP:
979     Prefix.setLowerBound(XOP);
980     break;
981   case X86II::VEX:
982     // VEX can be 2 byte or 3 byte, not determined yet if not explicit
983     Prefix.setLowerBound(MI.getFlags() & X86::IP_USE_VEX3 ? VEX3 : VEX2);
984     break;
985   case X86II::EVEX:
986     Prefix.setLowerBound(EVEX);
987     break;
988   }
989 
990   Prefix.setW(TSFlags & X86II::REX_W);
991   Prefix.setNF(TSFlags & X86II::EVEX_NF);
992 
993   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
994   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
995   bool IsND = X86II::hasNewDataDest(TSFlags); // IsND implies HasVEX_4V
996   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
997 
998   switch (TSFlags & X86II::OpMapMask) {
999   default:
1000     llvm_unreachable("Invalid prefix!");
1001   case X86II::TB:
1002     Prefix.set5M(0x1); // 0F
1003     break;
1004   case X86II::T8:
1005     Prefix.set5M(0x2); // 0F 38
1006     break;
1007   case X86II::TA:
1008     Prefix.set5M(0x3); // 0F 3A
1009     break;
1010   case X86II::XOP8:
1011     Prefix.set5M(0x8);
1012     break;
1013   case X86II::XOP9:
1014     Prefix.set5M(0x9);
1015     break;
1016   case X86II::XOPA:
1017     Prefix.set5M(0xA);
1018     break;
1019   case X86II::T_MAP4:
1020     Prefix.set5M(0x4);
1021     break;
1022   case X86II::T_MAP5:
1023     Prefix.set5M(0x5);
1024     break;
1025   case X86II::T_MAP6:
1026     Prefix.set5M(0x6);
1027     break;
1028   case X86II::T_MAP7:
1029     Prefix.set5M(0x7);
1030     break;
1031   }
1032 
1033   Prefix.setL(TSFlags & X86II::VEX_L);
1034   Prefix.setL2(TSFlags & X86II::EVEX_L2);
1035   if ((TSFlags & X86II::EVEX_L2) && STI.hasFeature(X86::FeatureAVX512) &&
1036       !STI.hasFeature(X86::FeatureEVEX512))
1037     report_fatal_error("ZMM registers are not supported without EVEX512");
1038   switch (TSFlags & X86II::OpPrefixMask) {
1039   case X86II::PD:
1040     Prefix.setPP(0x1); // 66
1041     break;
1042   case X86II::XS:
1043     Prefix.setPP(0x2); // F3
1044     break;
1045   case X86II::XD:
1046     Prefix.setPP(0x3); // F2
1047     break;
1048   }
1049 
1050   Prefix.setZ(HasEVEX_K && (TSFlags & X86II::EVEX_Z));
1051   Prefix.setEVEX_b(TSFlags & X86II::EVEX_B);
1052 
1053   bool EncodeRC = false;
1054   uint8_t EVEX_rc = 0;
1055 
1056   unsigned CurOp = X86II::getOperandBias(Desc);
1057 
1058   switch (TSFlags & X86II::FormMask) {
1059   default:
1060     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
1061   case X86II::MRMDestMem4VOp3CC: {
1062     //  src1(ModR/M), MemAddr, src2(VEX_4V)
1063     Prefix.setRR2(MI, CurOp++);
1064     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1065     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1066     CurOp += X86::AddrNumOperands;
1067     Prefix.set4VV2(MI, CurOp++);
1068     break;
1069   }
1070   case X86II::MRM_C0:
1071   case X86II::RawFrm:
1072     break;
1073   case X86II::MRMDestMemFSIB:
1074   case X86II::MRMDestMem: {
1075     // MRMDestMem instructions forms:
1076     //  MemAddr, src1(ModR/M)
1077     //  MemAddr, src1(VEX_4V), src2(ModR/M)
1078     //  MemAddr, src1(ModR/M), imm8
1079     //
1080     // NDD:
1081     //  dst(VEX_4V), MemAddr, src1(ModR/M)
1082     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1083     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1084     Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);
1085 
1086     if (IsND)
1087       Prefix.set4VV2(MI, CurOp++);
1088 
1089     CurOp += X86::AddrNumOperands;
1090 
1091     if (HasEVEX_K)
1092       Prefix.setAAA(MI, CurOp++);
1093 
1094     if (!IsND && HasVEX_4V)
1095       Prefix.set4VV2(MI, CurOp++);
1096 
1097     Prefix.setRR2(MI, CurOp++);
1098     break;
1099   }
1100   case X86II::MRMSrcMemFSIB:
1101   case X86II::MRMSrcMem: {
1102     // MRMSrcMem instructions forms:
1103     //  src1(ModR/M), MemAddr
1104     //  src1(ModR/M), src2(VEX_4V), MemAddr
1105     //  src1(ModR/M), MemAddr, imm8
1106     //  src1(ModR/M), MemAddr, src2(Imm[7:4])
1107     //
1108     //  FMA4:
1109     //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1110     //
1111     //  NDD:
1112     //  dst(VEX_4V), src1(ModR/M), MemAddr
1113     if (IsND)
1114       Prefix.set4VV2(MI, CurOp++);
1115 
1116     Prefix.setRR2(MI, CurOp++);
1117 
1118     if (HasEVEX_K)
1119       Prefix.setAAA(MI, CurOp++);
1120 
1121     if (!IsND && HasVEX_4V)
1122       Prefix.set4VV2(MI, CurOp++);
1123 
1124     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1125     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1126     Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);
1127 
1128     break;
1129   }
1130   case X86II::MRMSrcMem4VOp3: {
1131     // Instruction format for 4VOp3:
1132     //   src1(ModR/M), MemAddr, src3(VEX_4V)
1133     Prefix.setRR2(MI, CurOp++);
1134     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1135     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1136     Prefix.set4VV2(MI, CurOp + X86::AddrNumOperands);
1137     break;
1138   }
1139   case X86II::MRMSrcMemOp4: {
1140     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1141     Prefix.setR(MI, CurOp++);
1142     Prefix.set4V(MI, CurOp++);
1143     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1144     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1145     break;
1146   }
1147   case X86II::MRM0m:
1148   case X86II::MRM1m:
1149   case X86II::MRM2m:
1150   case X86II::MRM3m:
1151   case X86II::MRM4m:
1152   case X86II::MRM5m:
1153   case X86II::MRM6m:
1154   case X86II::MRM7m: {
1155     // MRM[0-9]m instructions forms:
1156     //  MemAddr
1157     //  src1(VEX_4V), MemAddr
1158     if (HasVEX_4V)
1159       Prefix.set4VV2(MI, CurOp++);
1160 
1161     if (HasEVEX_K)
1162       Prefix.setAAA(MI, CurOp++);
1163 
1164     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1165     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1166     Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);
1167 
1168     break;
1169   }
1170   case X86II::MRMSrcReg: {
1171     // MRMSrcReg instructions forms:
1172     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1173     //  dst(ModR/M), src1(ModR/M)
1174     //  dst(ModR/M), src1(ModR/M), imm8
1175     //
1176     //  FMA4:
1177     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1178     //
1179     //  NDD:
1180     //  dst(VEX_4V), src1(ModR/M.reg), src2(ModR/M)
1181     if (IsND)
1182       Prefix.set4VV2(MI, CurOp++);
1183     Prefix.setRR2(MI, CurOp++);
1184 
1185     if (HasEVEX_K)
1186       Prefix.setAAA(MI, CurOp++);
1187 
1188     if (!IsND && HasVEX_4V)
1189       Prefix.set4VV2(MI, CurOp++);
1190 
1191     Prefix.setBB2(MI, CurOp);
1192     Prefix.setX(MI, CurOp, 4);
1193     ++CurOp;
1194 
1195     if (TSFlags & X86II::EVEX_B) {
1196       if (HasEVEX_RC) {
1197         unsigned NumOps = Desc.getNumOperands();
1198         unsigned RcOperand = NumOps - 1;
1199         assert(RcOperand >= CurOp);
1200         EVEX_rc = MI.getOperand(RcOperand).getImm();
1201         assert(EVEX_rc <= 3 && "Invalid rounding control!");
1202       }
1203       EncodeRC = true;
1204     }
1205     break;
1206   }
1207   case X86II::MRMSrcReg4VOp3: {
1208     // Instruction format for 4VOp3:
1209     //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1210     Prefix.setRR2(MI, CurOp++);
1211     Prefix.setBB2(MI, CurOp++);
1212     Prefix.set4VV2(MI, CurOp++);
1213     break;
1214   }
1215   case X86II::MRMSrcRegOp4: {
1216     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1217     Prefix.setR(MI, CurOp++);
1218     Prefix.set4V(MI, CurOp++);
1219     // Skip second register source (encoded in Imm[7:4])
1220     ++CurOp;
1221 
1222     Prefix.setB(MI, CurOp);
1223     Prefix.setX(MI, CurOp, 4);
1224     ++CurOp;
1225     break;
1226   }
1227   case X86II::MRMDestReg: {
1228     // MRMDestReg instructions forms:
1229     //  dst(ModR/M), src(ModR/M)
1230     //  dst(ModR/M), src(ModR/M), imm8
1231     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1232     //
1233     // NDD:
1234     // dst(VEX_4V), src1(ModR/M), src2(ModR/M)
1235     if (IsND)
1236       Prefix.set4VV2(MI, CurOp++);
1237     Prefix.setBB2(MI, CurOp);
1238     Prefix.setX(MI, CurOp, 4);
1239     ++CurOp;
1240 
1241     if (HasEVEX_K)
1242       Prefix.setAAA(MI, CurOp++);
1243 
1244     if (!IsND && HasVEX_4V)
1245       Prefix.set4VV2(MI, CurOp++);
1246 
1247     Prefix.setRR2(MI, CurOp++);
1248     if (TSFlags & X86II::EVEX_B)
1249       EncodeRC = true;
1250     break;
1251   }
1252   case X86II::MRMr0: {
1253     // MRMr0 instructions forms:
1254     //  11:rrr:000
1255     //  dst(ModR/M)
1256     Prefix.setRR2(MI, CurOp++);
1257     break;
1258   }
1259   case X86II::MRM0r:
1260   case X86II::MRM1r:
1261   case X86II::MRM2r:
1262   case X86II::MRM3r:
1263   case X86II::MRM4r:
1264   case X86II::MRM5r:
1265   case X86II::MRM6r:
1266   case X86II::MRM7r: {
1267     // MRM0r-MRM7r instructions forms:
1268     //  dst(VEX_4V), src(ModR/M), imm8
1269     if (HasVEX_4V)
1270       Prefix.set4VV2(MI, CurOp++);
1271 
1272     if (HasEVEX_K)
1273       Prefix.setAAA(MI, CurOp++);
1274 
1275     Prefix.setBB2(MI, CurOp);
1276     Prefix.setX(MI, CurOp, 4);
1277     ++CurOp;
1278     break;
1279   }
1280   }
1281   if (EncodeRC) {
1282     Prefix.setL(EVEX_rc & 0x1);
1283     Prefix.setL2(EVEX_rc & 0x2);
1284   }
1285   PrefixKind Kind = Prefix.determineOptimalKind();
1286   Prefix.emit(CB);
1287   return Kind;
1288 }
1289 
1290 /// Emit REX prefix which specifies
1291 ///   1) 64-bit instructions,
1292 ///   2) non-default operand size, and
1293 ///   3) use of X86-64 extended registers.
1294 ///
1295 /// \returns the used prefix (REX or None).
1296 PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1297                                            const MCSubtargetInfo &STI,
1298                                            SmallVectorImpl<char> &CB) const {
1299   if (!STI.hasFeature(X86::Is64Bit))
1300     return None;
1301   X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());
1302   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1303   uint64_t TSFlags = Desc.TSFlags;
1304   Prefix.setW(TSFlags & X86II::REX_W);
1305   unsigned NumOps = MI.getNumOperands();
1306   bool UsesHighByteReg = false;
1307 #ifndef NDEBUG
1308   bool HasRegOp = false;
1309 #endif
1310   unsigned CurOp = NumOps ? X86II::getOperandBias(Desc) : 0;
1311   for (unsigned i = CurOp; i != NumOps; ++i) {
1312     const MCOperand &MO = MI.getOperand(i);
1313     if (MO.isReg()) {
1314 #ifndef NDEBUG
1315       HasRegOp = true;
1316 #endif
1317       unsigned Reg = MO.getReg();
1318       if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1319         UsesHighByteReg = true;
1320       // If it accesses SPL, BPL, SIL, or DIL, then it requires a REX prefix.
1321       if (X86II::isX86_64NonExtLowByteReg(Reg))
1322         Prefix.setLowerBound(REX);
1323     } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {
1324       // GOTTPOFF and TLSDESC relocations require a REX prefix to allow
1325       // linker optimizations: even if the instructions we see may not require
1326       // any prefix, they may be replaced by instructions that do. This is
1327       // handled as a special case here so that it also works for hand-written
1328       // assembly without the user needing to write REX, as with GNU as.
1329       const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
1330       if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF ||
1331                   Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) {
1332         Prefix.setLowerBound(REX);
1333       }
1334     }
1335   }
1336   if ((TSFlags & X86II::ExplicitOpPrefixMask) == X86II::ExplicitREX2Prefix)
1337     Prefix.setLowerBound(REX2);
1338   switch (TSFlags & X86II::FormMask) {
1339   default:
1340     assert(!HasRegOp && "Unexpected form in emitREXPrefix!");
1341     break;
1342   case X86II::RawFrm:
1343   case X86II::RawFrmMemOffs:
1344   case X86II::RawFrmSrc:
1345   case X86II::RawFrmDst:
1346   case X86II::RawFrmDstSrc:
1347     break;
1348   case X86II::AddRegFrm:
1349     Prefix.setBB2(MI, CurOp++);
1350     break;
1351   case X86II::MRMSrcReg:
1352   case X86II::MRMSrcRegCC:
1353     Prefix.setRR2(MI, CurOp++);
1354     Prefix.setBB2(MI, CurOp++);
1355     break;
1356   case X86II::MRMSrcMem:
1357   case X86II::MRMSrcMemCC:
1358     Prefix.setRR2(MI, CurOp++);
1359     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1360     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1361     CurOp += X86::AddrNumOperands;
1362     break;
1363   case X86II::MRMDestReg:
1364     Prefix.setBB2(MI, CurOp++);
1365     Prefix.setRR2(MI, CurOp++);
1366     break;
1367   case X86II::MRMDestMem:
1368     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1369     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1370     CurOp += X86::AddrNumOperands;
1371     Prefix.setRR2(MI, CurOp++);
1372     break;
1373   case X86II::MRMXmCC:
1374   case X86II::MRMXm:
1375   case X86II::MRM0m:
1376   case X86II::MRM1m:
1377   case X86II::MRM2m:
1378   case X86II::MRM3m:
1379   case X86II::MRM4m:
1380   case X86II::MRM5m:
1381   case X86II::MRM6m:
1382   case X86II::MRM7m:
1383     Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);
1384     Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);
1385     break;
1386   case X86II::MRMXrCC:
1387   case X86II::MRMXr:
1388   case X86II::MRM0r:
1389   case X86II::MRM1r:
1390   case X86II::MRM2r:
1391   case X86II::MRM3r:
1392   case X86II::MRM4r:
1393   case X86II::MRM5r:
1394   case X86II::MRM6r:
1395   case X86II::MRM7r:
1396     Prefix.setBB2(MI, CurOp++);
1397     break;
1398   }
1399   Prefix.setM((TSFlags & X86II::OpMapMask) == X86II::TB);
1400   PrefixKind Kind = Prefix.determineOptimalKind();
1401   if (Kind && UsesHighByteReg)
1402     report_fatal_error(
1403         "Cannot encode high byte register in REX-prefixed instruction");
1404   Prefix.emit(CB);
1405   return Kind;
1406 }
1407 
1408 /// Emit segment override opcode prefix as needed.
1409 void X86MCCodeEmitter::emitSegmentOverridePrefix(
1410     unsigned SegOperand, const MCInst &MI, SmallVectorImpl<char> &CB) const {
1411   // Check for explicit segment override on memory operand.
1412   if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1413     emitByte(X86::getSegmentOverridePrefixForReg(Reg), CB);
1414 }
1415 
1416 /// Emit all instruction prefixes prior to the opcode.
1417 ///
1418 /// \param MemOperand the operand # of the start of a memory operand if present.
1419 /// If not present, it is -1.
1420 ///
1421 /// \returns the used prefix (REX or None).
1422 PrefixKind X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1423                                               const MCSubtargetInfo &STI,
1424                                               SmallVectorImpl<char> &CB) const {
1425   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1426   uint64_t TSFlags = Desc.TSFlags;
1427 
1428   // Emit the operand size opcode prefix as needed.
1429   if ((TSFlags & X86II::OpSizeMask) ==
1430       (STI.hasFeature(X86::Is16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1431     emitByte(0x66, CB);
1432 
1433   // Emit the LOCK opcode prefix.
1434   if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1435     emitByte(0xF0, CB);
1436 
1437   // Emit the NOTRACK opcode prefix.
1438   if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1439     emitByte(0x3E, CB);
1440 
1441   switch (TSFlags & X86II::OpPrefixMask) {
1442   case X86II::PD: // 66
1443     emitByte(0x66, CB);
1444     break;
1445   case X86II::XS: // F3
1446     emitByte(0xF3, CB);
1447     break;
1448   case X86II::XD: // F2
1449     emitByte(0xF2, CB);
1450     break;
1451   }
1452 
1453   // Handle REX prefix.
1454   assert((STI.hasFeature(X86::Is64Bit) || !(TSFlags & X86II::REX_W)) &&
1455          "REX.W requires 64bit mode.");
1456   PrefixKind Kind = emitREXPrefix(MemOperand, MI, STI, CB);
1457 
1458   // 0x0F escape code must be emitted just before the opcode.
1459   switch (TSFlags & X86II::OpMapMask) {
1460   case X86II::TB:        // Two-byte opcode map
1461     // Encoded by M bit in REX2
1462     if (Kind == REX2)
1463       break;
1464     [[fallthrough]];
1465   case X86II::T8:        // 0F 38
1466   case X86II::TA:        // 0F 3A
1467   case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1468     emitByte(0x0F, CB);
1469     break;
1470   }
1471 
1472   switch (TSFlags & X86II::OpMapMask) {
1473   case X86II::T8: // 0F 38
1474     emitByte(0x38, CB);
1475     break;
1476   case X86II::TA: // 0F 3A
1477     emitByte(0x3A, CB);
1478     break;
1479   }
1480 
1481   return Kind;
1482 }
1483 
1484 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,
1485                                   const MCSubtargetInfo &STI) const {
1486   unsigned Opcode = MI.getOpcode();
1487   const MCInstrDesc &Desc = MCII.get(Opcode);
1488   uint64_t TSFlags = Desc.TSFlags;
1489 
1490   // Pseudo instructions don't get encoded.
1491   if (X86II::isPseudo(TSFlags))
1492     return;
1493 
1494   unsigned CurOp = X86II::getOperandBias(Desc);
1495 
1496   emitPrefixImpl(CurOp, MI, STI, CB);
1497 }
1498 
1499 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
1500                                          SmallVectorImpl<char> &CB,
1501                                          SmallVectorImpl<MCFixup> &Fixups,
1502                                          const MCSubtargetInfo &STI) const {
1503   unsigned Opcode = MI.getOpcode();
1504   const MCInstrDesc &Desc = MCII.get(Opcode);
1505   uint64_t TSFlags = Desc.TSFlags;
1506 
1507   // Pseudo instructions don't get encoded.
1508   if (X86II::isPseudo(TSFlags))
1509     return;
1510 
1511   unsigned NumOps = Desc.getNumOperands();
1512   unsigned CurOp = X86II::getOperandBias(Desc);
1513 
1514   uint64_t StartByte = CB.size();
1515 
1516   PrefixKind Kind = emitPrefixImpl(CurOp, MI, STI, CB);
1517 
1518   // It uses the VEX.VVVV field?
1519   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1520   bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1521 
1522   // It uses the EVEX.aaa field?
1523   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1524   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1525 
1526   // Used if a register is encoded in 7:4 of immediate.
1527   unsigned I8RegNum = 0;
1528 
1529   uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1530 
1531   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1532     BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1533 
1534   unsigned OpcodeOffset = 0;
1535 
1536   bool IsND = X86II::hasNewDataDest(TSFlags);
1537 
1538   uint64_t Form = TSFlags & X86II::FormMask;
1539   switch (Form) {
1540   default:
1541     errs() << "FORM: " << Form << "\n";
1542     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1543   case X86II::Pseudo:
1544     llvm_unreachable("Pseudo instruction shouldn't be emitted");
1545   case X86II::RawFrmDstSrc:
1546   case X86II::RawFrmSrc:
1547   case X86II::RawFrmDst:
1548   case X86II::PrefixByte:
1549     emitByte(BaseOpcode, CB);
1550     break;
1551   case X86II::AddCCFrm: {
1552     // This will be added to the opcode in the fallthrough.
1553     OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1554     assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1555     --NumOps; // Drop the operand from the end.
1556     [[fallthrough]];
1557   case X86II::RawFrm:
1558     emitByte(BaseOpcode + OpcodeOffset, CB);
1559 
1560     if (!STI.hasFeature(X86::Is64Bit) || !isPCRel32Branch(MI, MCII))
1561       break;
1562 
1563     const MCOperand &Op = MI.getOperand(CurOp++);
1564     emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1565                   MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, CB,
1566                   Fixups);
1567     break;
1568   }
1569   case X86II::RawFrmMemOffs:
1570     emitByte(BaseOpcode, CB);
1571     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1572                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1573                   StartByte, CB, Fixups);
1574     ++CurOp; // skip segment operand
1575     break;
1576   case X86II::RawFrmImm8:
1577     emitByte(BaseOpcode, CB);
1578     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1579                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1580                   StartByte, CB, Fixups);
1581     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1582                   CB, Fixups);
1583     break;
1584   case X86II::RawFrmImm16:
1585     emitByte(BaseOpcode, CB);
1586     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1587                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1588                   StartByte, CB, Fixups);
1589     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1590                   CB, Fixups);
1591     break;
1592 
1593   case X86II::AddRegFrm:
1594     emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), CB);
1595     break;
1596 
1597   case X86II::MRMDestReg: {
1598     emitByte(BaseOpcode, CB);
1599     unsigned SrcRegNum = CurOp + 1;
1600 
1601     if (HasEVEX_K) // Skip writemask
1602       ++SrcRegNum;
1603 
1604     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1605       ++SrcRegNum;
1606     if (IsND) // Skip the NDD operand encoded in EVEX_VVVV
1607       ++CurOp;
1608 
1609     emitRegModRMByte(MI.getOperand(CurOp),
1610                      getX86RegNum(MI.getOperand(SrcRegNum)), CB);
1611     CurOp = SrcRegNum + 1;
1612     break;
1613   }
1614   case X86II::MRMDestMem4VOp3CC: {
1615     unsigned CC = MI.getOperand(8).getImm();
1616     emitByte(BaseOpcode + CC, CB);
1617     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1618     emitMemModRMByte(MI, CurOp + 1, getX86RegNum(MI.getOperand(0)), TSFlags,
1619                      Kind, StartByte, CB, Fixups, STI, false);
1620     CurOp = SrcRegNum + 3; // skip reg, VEX_V4 and CC
1621     break;
1622   }
1623   case X86II::MRMDestMemFSIB:
1624   case X86II::MRMDestMem: {
1625     emitByte(BaseOpcode, CB);
1626     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1627 
1628     if (HasEVEX_K) // Skip writemask
1629       ++SrcRegNum;
1630 
1631     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1632       ++SrcRegNum;
1633 
1634     if (IsND) // Skip new data destination
1635       ++CurOp;
1636 
1637     bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1638     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1639                      Kind, StartByte, CB, Fixups, STI, ForceSIB);
1640     CurOp = SrcRegNum + 1;
1641     break;
1642   }
1643   case X86II::MRMSrcReg: {
1644     emitByte(BaseOpcode, CB);
1645     unsigned SrcRegNum = CurOp + 1;
1646 
1647     if (HasEVEX_K) // Skip writemask
1648       ++SrcRegNum;
1649 
1650     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1651       ++SrcRegNum;
1652 
1653     if (IsND) // Skip new data destination
1654       ++CurOp;
1655 
1656     emitRegModRMByte(MI.getOperand(SrcRegNum),
1657                      getX86RegNum(MI.getOperand(CurOp)), CB);
1658     CurOp = SrcRegNum + 1;
1659     if (HasVEX_I8Reg)
1660       I8RegNum = getX86RegEncoding(MI, CurOp++);
1661     // do not count the rounding control operand
1662     if (HasEVEX_RC)
1663       --NumOps;
1664     break;
1665   }
1666   case X86II::MRMSrcReg4VOp3: {
1667     emitByte(BaseOpcode, CB);
1668     unsigned SrcRegNum = CurOp + 1;
1669 
1670     emitRegModRMByte(MI.getOperand(SrcRegNum),
1671                      getX86RegNum(MI.getOperand(CurOp)), CB);
1672     CurOp = SrcRegNum + 1;
1673     ++CurOp; // Encoded in VEX.VVVV
1674     break;
1675   }
1676   case X86II::MRMSrcRegOp4: {
1677     emitByte(BaseOpcode, CB);
1678     unsigned SrcRegNum = CurOp + 1;
1679 
1680     // Skip 1st src (which is encoded in VEX_VVVV)
1681     ++SrcRegNum;
1682 
1683     // Capture 2nd src (which is encoded in Imm[7:4])
1684     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1685     I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1686 
1687     emitRegModRMByte(MI.getOperand(SrcRegNum),
1688                      getX86RegNum(MI.getOperand(CurOp)), CB);
1689     CurOp = SrcRegNum + 1;
1690     break;
1691   }
1692   case X86II::MRMSrcRegCC: {
1693     unsigned FirstOp = CurOp++;
1694     unsigned SecondOp = CurOp++;
1695 
1696     unsigned CC = MI.getOperand(CurOp++).getImm();
1697     emitByte(BaseOpcode + CC, CB);
1698 
1699     emitRegModRMByte(MI.getOperand(SecondOp),
1700                      getX86RegNum(MI.getOperand(FirstOp)), CB);
1701     break;
1702   }
1703   case X86II::MRMSrcMemFSIB:
1704   case X86II::MRMSrcMem: {
1705     unsigned FirstMemOp = CurOp + 1;
1706 
1707     if (IsND) // Skip new data destination
1708       CurOp++;
1709 
1710     if (HasEVEX_K) // Skip writemask
1711       ++FirstMemOp;
1712 
1713     if (HasVEX_4V)
1714       ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1715 
1716     emitByte(BaseOpcode, CB);
1717 
1718     bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1719     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1720                      TSFlags, Kind, StartByte, CB, Fixups, STI, ForceSIB);
1721     CurOp = FirstMemOp + X86::AddrNumOperands;
1722     if (HasVEX_I8Reg)
1723       I8RegNum = getX86RegEncoding(MI, CurOp++);
1724     break;
1725   }
1726   case X86II::MRMSrcMem4VOp3: {
1727     unsigned FirstMemOp = CurOp + 1;
1728 
1729     emitByte(BaseOpcode, CB);
1730 
1731     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1732                      TSFlags, Kind, StartByte, CB, Fixups, STI);
1733     CurOp = FirstMemOp + X86::AddrNumOperands;
1734     ++CurOp; // Encoded in VEX.VVVV.
1735     break;
1736   }
1737   case X86II::MRMSrcMemOp4: {
1738     unsigned FirstMemOp = CurOp + 1;
1739 
1740     ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1741 
1742     // Capture second register source (encoded in Imm[7:4])
1743     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1744     I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1745 
1746     emitByte(BaseOpcode, CB);
1747 
1748     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1749                      TSFlags, Kind, StartByte, CB, Fixups, STI);
1750     CurOp = FirstMemOp + X86::AddrNumOperands;
1751     break;
1752   }
1753   case X86II::MRMSrcMemCC: {
1754     unsigned RegOp = CurOp++;
1755     unsigned FirstMemOp = CurOp;
1756     CurOp = FirstMemOp + X86::AddrNumOperands;
1757 
1758     unsigned CC = MI.getOperand(CurOp++).getImm();
1759     emitByte(BaseOpcode + CC, CB);
1760 
1761     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1762                      TSFlags, Kind, StartByte, CB, Fixups, STI);
1763     break;
1764   }
1765 
1766   case X86II::MRMXrCC: {
1767     unsigned RegOp = CurOp++;
1768 
1769     unsigned CC = MI.getOperand(CurOp++).getImm();
1770     emitByte(BaseOpcode + CC, CB);
1771     emitRegModRMByte(MI.getOperand(RegOp), 0, CB);
1772     break;
1773   }
1774 
1775   case X86II::MRMXr:
1776   case X86II::MRM0r:
1777   case X86II::MRM1r:
1778   case X86II::MRM2r:
1779   case X86II::MRM3r:
1780   case X86II::MRM4r:
1781   case X86II::MRM5r:
1782   case X86II::MRM6r:
1783   case X86II::MRM7r:
1784     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1785       ++CurOp;
1786     if (HasEVEX_K) // Skip writemask
1787       ++CurOp;
1788     emitByte(BaseOpcode, CB);
1789     emitRegModRMByte(MI.getOperand(CurOp++),
1790                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CB);
1791     break;
1792   case X86II::MRMr0:
1793     emitByte(BaseOpcode, CB);
1794     emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)), 0), CB);
1795     break;
1796 
1797   case X86II::MRMXmCC: {
1798     unsigned FirstMemOp = CurOp;
1799     CurOp = FirstMemOp + X86::AddrNumOperands;
1800 
1801     unsigned CC = MI.getOperand(CurOp++).getImm();
1802     emitByte(BaseOpcode + CC, CB);
1803 
1804     emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, Kind, StartByte, CB, Fixups,
1805                      STI);
1806     break;
1807   }
1808 
1809   case X86II::MRMXm:
1810   case X86II::MRM0m:
1811   case X86II::MRM1m:
1812   case X86II::MRM2m:
1813   case X86II::MRM3m:
1814   case X86II::MRM4m:
1815   case X86II::MRM5m:
1816   case X86II::MRM6m:
1817   case X86II::MRM7m:
1818     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1819       ++CurOp;
1820     if (HasEVEX_K) // Skip writemask
1821       ++CurOp;
1822     emitByte(BaseOpcode, CB);
1823     emitMemModRMByte(MI, CurOp,
1824                      (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1825                      Kind, StartByte, CB, Fixups, STI);
1826     CurOp += X86::AddrNumOperands;
1827     break;
1828 
1829   case X86II::MRM0X:
1830   case X86II::MRM1X:
1831   case X86II::MRM2X:
1832   case X86II::MRM3X:
1833   case X86II::MRM4X:
1834   case X86II::MRM5X:
1835   case X86II::MRM6X:
1836   case X86II::MRM7X:
1837     emitByte(BaseOpcode, CB);
1838     emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), CB);
1839     break;
1840 
1841   case X86II::MRM_C0:
1842   case X86II::MRM_C1:
1843   case X86II::MRM_C2:
1844   case X86II::MRM_C3:
1845   case X86II::MRM_C4:
1846   case X86II::MRM_C5:
1847   case X86II::MRM_C6:
1848   case X86II::MRM_C7:
1849   case X86II::MRM_C8:
1850   case X86II::MRM_C9:
1851   case X86II::MRM_CA:
1852   case X86II::MRM_CB:
1853   case X86II::MRM_CC:
1854   case X86II::MRM_CD:
1855   case X86II::MRM_CE:
1856   case X86II::MRM_CF:
1857   case X86II::MRM_D0:
1858   case X86II::MRM_D1:
1859   case X86II::MRM_D2:
1860   case X86II::MRM_D3:
1861   case X86II::MRM_D4:
1862   case X86II::MRM_D5:
1863   case X86II::MRM_D6:
1864   case X86II::MRM_D7:
1865   case X86II::MRM_D8:
1866   case X86II::MRM_D9:
1867   case X86II::MRM_DA:
1868   case X86II::MRM_DB:
1869   case X86II::MRM_DC:
1870   case X86II::MRM_DD:
1871   case X86II::MRM_DE:
1872   case X86II::MRM_DF:
1873   case X86II::MRM_E0:
1874   case X86II::MRM_E1:
1875   case X86II::MRM_E2:
1876   case X86II::MRM_E3:
1877   case X86II::MRM_E4:
1878   case X86II::MRM_E5:
1879   case X86II::MRM_E6:
1880   case X86II::MRM_E7:
1881   case X86II::MRM_E8:
1882   case X86II::MRM_E9:
1883   case X86II::MRM_EA:
1884   case X86II::MRM_EB:
1885   case X86II::MRM_EC:
1886   case X86II::MRM_ED:
1887   case X86II::MRM_EE:
1888   case X86II::MRM_EF:
1889   case X86II::MRM_F0:
1890   case X86II::MRM_F1:
1891   case X86II::MRM_F2:
1892   case X86II::MRM_F3:
1893   case X86II::MRM_F4:
1894   case X86II::MRM_F5:
1895   case X86II::MRM_F6:
1896   case X86II::MRM_F7:
1897   case X86II::MRM_F8:
1898   case X86II::MRM_F9:
1899   case X86II::MRM_FA:
1900   case X86II::MRM_FB:
1901   case X86II::MRM_FC:
1902   case X86II::MRM_FD:
1903   case X86II::MRM_FE:
1904   case X86II::MRM_FF:
1905     emitByte(BaseOpcode, CB);
1906     emitByte(0xC0 + Form - X86II::MRM_C0, CB);
1907     break;
1908   }
1909 
1910   if (HasVEX_I8Reg) {
1911     // The last source register of a 4 operand instruction in AVX is encoded
1912     // in bits[7:4] of a immediate byte.
1913     assert(I8RegNum < 16 && "Register encoding out of range");
1914     I8RegNum <<= 4;
1915     if (CurOp != NumOps) {
1916       unsigned Val = MI.getOperand(CurOp++).getImm();
1917       assert(Val < 16 && "Immediate operand value out of range");
1918       I8RegNum |= Val;
1919     }
1920     emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1921                   StartByte, CB, Fixups);
1922   } else {
1923     // If there is a remaining operand, it must be a trailing immediate. Emit it
1924     // according to the right size for the instruction. Some instructions
1925     // (SSE4a extrq and insertq) have two trailing immediates.
1926     while (CurOp != NumOps && NumOps - CurOp <= 2) {
1927       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1928                     X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1929                     StartByte, CB, Fixups);
1930     }
1931   }
1932 
1933   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1934     emitByte(X86II::getBaseOpcodeFor(TSFlags), CB);
1935 
1936   assert(CB.size() - StartByte <= 15 &&
1937          "The size of instruction must be no longer than 15.");
1938 #ifndef NDEBUG
1939   // FIXME: Verify.
1940   if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1941     errs() << "Cannot encode all operands of: ";
1942     MI.dump();
1943     errs() << '\n';
1944     abort();
1945   }
1946 #endif
1947 }
1948 
1949 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1950                                             MCContext &Ctx) {
1951   return new X86MCCodeEmitter(MCII, Ctx);
1952 }
1953