1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "mccodeemitter"
36 
37 namespace {
38 
39 class X86MCCodeEmitter : public MCCodeEmitter {
40   const MCInstrInfo &MCII;
41   MCContext &Ctx;
42 
43 public:
44   X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45       : MCII(mcii), Ctx(ctx) {}
46   X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47   X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48   ~X86MCCodeEmitter() override = default;
49 
50   void emitPrefix(const MCInst &MI, raw_ostream &OS,
51                   const MCSubtargetInfo &STI) const override;
52 
53   void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54                          SmallVectorImpl<MCFixup> &Fixups,
55                          const MCSubtargetInfo &STI) const override;
56 
57 private:
58   unsigned getX86RegNum(const MCOperand &MO) const;
59 
60   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61 
62   /// \param MI a single low-level machine instruction.
63   /// \param OpNum the operand #.
64   /// \returns true if the OpNumth operand of MI  require a bit to be set in
65   /// REX prefix.
66   bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67 
68   void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69                      MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
70                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71 
72   void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73                         raw_ostream &OS) const;
74 
75   void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76                    raw_ostream &OS) const;
77 
78   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79                         uint64_t TSFlags, bool HasREX, uint64_t StartByte,
80                         raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81                         const MCSubtargetInfo &STI,
82                         bool ForceSIB = false) const;
83 
84   bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
85                       const MCSubtargetInfo &STI, raw_ostream &OS) const;
86 
87   void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
88                            raw_ostream &OS) const;
89 
90   void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
91                                  raw_ostream &OS) const;
92 
93   bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
94                         const MCSubtargetInfo &STI, raw_ostream &OS) const;
95 
96   bool emitREXPrefix(int MemOperand, const MCInst &MI,
97                      const MCSubtargetInfo &STI, raw_ostream &OS) const;
98 };
99 
100 } // end anonymous namespace
101 
102 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
103   assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
104   return RM | (RegOpcode << 3) | (Mod << 6);
105 }
106 
107 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
108 
109 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
110   // Output the constant in little endian byte order.
111   for (unsigned i = 0; i != Size; ++i) {
112     emitByte(Val & 255, OS);
113     Val >>= 8;
114   }
115 }
116 
117 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
118 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
119 /// parameter of emitImmediate.
120 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
121   bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
122 
123   int CD8_Scale =
124       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
125   if (!HasEVEX || CD8_Scale == 0)
126     return isInt<8>(Value);
127 
128   assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
129   if (Value & (CD8_Scale - 1)) // Unaligned offset
130     return false;
131 
132   int CDisp8 = Value / CD8_Scale;
133   if (!isInt<8>(CDisp8))
134     return false;
135 
136   // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
137   ImmOffset = CDisp8 - Value;
138   return true;
139 }
140 
141 /// \returns the appropriate fixup kind to use for an immediate in an
142 /// instruction with the specified TSFlags.
143 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
144   unsigned Size = X86II::getSizeOfImm(TSFlags);
145   bool isPCRel = X86II::isImmPCRel(TSFlags);
146 
147   if (X86II::isImmSigned(TSFlags)) {
148     switch (Size) {
149     default:
150       llvm_unreachable("Unsupported signed fixup size!");
151     case 4:
152       return MCFixupKind(X86::reloc_signed_4byte);
153     }
154   }
155   return MCFixup::getKindForSize(Size, isPCRel);
156 }
157 
158 /// \param Op operand # of the memory operand.
159 ///
160 /// \returns true if the specified instruction has a 16-bit memory operand.
161 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
162                               const MCSubtargetInfo &STI) {
163   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
164   const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg);
165 
166   unsigned BaseReg = Base.getReg();
167   unsigned IndexReg = Index.getReg();
168 
169   if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0)
170     return true;
171   if ((BaseReg != 0 &&
172        X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) ||
173       (IndexReg != 0 &&
174        X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)))
175     return true;
176   return false;
177 }
178 
179 /// \param Op operand # of the memory operand.
180 ///
181 /// \returns true if the specified instruction has a 32-bit memory operand.
182 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
183   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
184   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
185 
186   if ((BaseReg.getReg() != 0 &&
187        X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
188       (IndexReg.getReg() != 0 &&
189        X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
190     return true;
191   if (BaseReg.getReg() == X86::EIP) {
192     assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
193     return true;
194   }
195   if (IndexReg.getReg() == X86::EIZ)
196     return true;
197   return false;
198 }
199 
200 /// \param Op operand # of the memory operand.
201 ///
202 /// \returns true if the specified instruction has a 64-bit memory operand.
203 #ifndef NDEBUG
204 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
205   const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
206   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
207 
208   if ((BaseReg.getReg() != 0 &&
209        X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
210       (IndexReg.getReg() != 0 &&
211        X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
212     return true;
213   return false;
214 }
215 #endif
216 
217 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
218 
219 /// Check if this expression starts with  _GLOBAL_OFFSET_TABLE_ and if it is
220 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
221 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
222 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
223 /// binary expression.
224 static GlobalOffsetTableExprKind
225 startsWithGlobalOffsetTable(const MCExpr *Expr) {
226   const MCExpr *RHS = nullptr;
227   if (Expr->getKind() == MCExpr::Binary) {
228     const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
229     Expr = BE->getLHS();
230     RHS = BE->getRHS();
231   }
232 
233   if (Expr->getKind() != MCExpr::SymbolRef)
234     return GOT_None;
235 
236   const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
237   const MCSymbol &S = Ref->getSymbol();
238   if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
239     return GOT_None;
240   if (RHS && RHS->getKind() == MCExpr::SymbolRef)
241     return GOT_SymDiff;
242   return GOT_Normal;
243 }
244 
245 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
246   if (Expr->getKind() == MCExpr::SymbolRef) {
247     const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
248     return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
249   }
250   return false;
251 }
252 
253 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
254   unsigned Opcode = MI.getOpcode();
255   const MCInstrDesc &Desc = MCII.get(Opcode);
256   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
257        Opcode != X86::JCC_4) ||
258       getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
259     return false;
260 
261   unsigned CurOp = X86II::getOperandBias(Desc);
262   const MCOperand &Op = MI.getOperand(CurOp);
263   if (!Op.isExpr())
264     return false;
265 
266   const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
267   return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
268 }
269 
270 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
271   return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
272 }
273 
274 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
275                                              unsigned OpNum) const {
276   return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
277 }
278 
279 /// \param MI a single low-level machine instruction.
280 /// \param OpNum the operand #.
281 /// \returns true if the OpNumth operand of MI  require a bit to be set in
282 /// REX prefix.
283 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
284                                         unsigned OpNum) const {
285   return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
286 }
287 
288 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
289                                      unsigned Size, MCFixupKind FixupKind,
290                                      uint64_t StartByte, raw_ostream &OS,
291                                      SmallVectorImpl<MCFixup> &Fixups,
292                                      int ImmOffset) const {
293   const MCExpr *Expr = nullptr;
294   if (DispOp.isImm()) {
295     // If this is a simple integer displacement that doesn't require a
296     // relocation, emit it now.
297     if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
298         FixupKind != FK_PCRel_4) {
299       emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
300       return;
301     }
302     Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
303   } else {
304     Expr = DispOp.getExpr();
305   }
306 
307   // If we have an immoffset, add it to the expression.
308   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
309        FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
310     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
311     if (Kind != GOT_None) {
312       assert(ImmOffset == 0);
313 
314       if (Size == 8) {
315         FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
316       } else {
317         assert(Size == 4);
318         FixupKind = MCFixupKind(X86::reloc_global_offset_table);
319       }
320 
321       if (Kind == GOT_Normal)
322         ImmOffset = static_cast<int>(OS.tell() - StartByte);
323     } else if (Expr->getKind() == MCExpr::SymbolRef) {
324       if (hasSecRelSymbolRef(Expr)) {
325         FixupKind = MCFixupKind(FK_SecRel_4);
326       }
327     } else if (Expr->getKind() == MCExpr::Binary) {
328       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
329       if (hasSecRelSymbolRef(Bin->getLHS()) ||
330           hasSecRelSymbolRef(Bin->getRHS())) {
331         FixupKind = MCFixupKind(FK_SecRel_4);
332       }
333     }
334   }
335 
336   // If the fixup is pc-relative, we need to bias the value to be relative to
337   // the start of the field, not the end of the field.
338   if (FixupKind == FK_PCRel_4 ||
339       FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
340       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
341       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
342       FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
343       FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
344     ImmOffset -= 4;
345     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
346     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
347     // this needs to be a GOTPC32 relocation.
348     if (startsWithGlobalOffsetTable(Expr) != GOT_None)
349       FixupKind = MCFixupKind(X86::reloc_global_offset_table);
350   }
351   if (FixupKind == FK_PCRel_2)
352     ImmOffset -= 2;
353   if (FixupKind == FK_PCRel_1)
354     ImmOffset -= 1;
355 
356   if (ImmOffset)
357     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
358                                    Ctx);
359 
360   // Emit a symbolic constant as a fixup and 4 zeros.
361   Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
362                                    Expr, FixupKind, Loc));
363   emitConstant(0, Size, OS);
364 }
365 
366 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
367                                         unsigned RegOpcodeFld,
368                                         raw_ostream &OS) const {
369   emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
370 }
371 
372 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
373                                    raw_ostream &OS) const {
374   // SIB byte is in the same format as the modRMByte.
375   emitByte(modRMByte(SS, Index, Base), OS);
376 }
377 
378 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
379                                         unsigned RegOpcodeField,
380                                         uint64_t TSFlags, bool HasREX,
381                                         uint64_t StartByte, raw_ostream &OS,
382                                         SmallVectorImpl<MCFixup> &Fixups,
383                                         const MCSubtargetInfo &STI,
384                                         bool ForceSIB) const {
385   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
386   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
387   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
388   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
389   unsigned BaseReg = Base.getReg();
390 
391   // Handle %rip relative addressing.
392   if (BaseReg == X86::RIP ||
393       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
394     assert(STI.hasFeature(X86::Mode64Bit) &&
395            "Rip-relative addressing requires 64-bit mode");
396     assert(IndexReg.getReg() == 0 && !ForceSIB &&
397            "Invalid rip-relative address");
398     emitByte(modRMByte(0, RegOpcodeField, 5), OS);
399 
400     unsigned Opcode = MI.getOpcode();
401     unsigned FixupKind = [&]() {
402       // Enable relaxed relocation only for a MCSymbolRefExpr.  We cannot use a
403       // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
404       if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))
405         return X86::reloc_riprel_4byte;
406 
407       // Certain loads for GOT references can be relocated against the symbol
408       // directly if the symbol ends up in the same linkage unit.
409       switch (Opcode) {
410       default:
411         return X86::reloc_riprel_4byte;
412       case X86::MOV64rm:
413         // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a
414         // special case because COFF and Mach-O don't support ELF's more
415         // flexible R_X86_64_REX_GOTPCRELX relaxation.
416         assert(HasREX);
417         return X86::reloc_riprel_4byte_movq_load;
418       case X86::ADC32rm:
419       case X86::ADD32rm:
420       case X86::AND32rm:
421       case X86::CMP32rm:
422       case X86::MOV32rm:
423       case X86::OR32rm:
424       case X86::SBB32rm:
425       case X86::SUB32rm:
426       case X86::TEST32mr:
427       case X86::XOR32rm:
428       case X86::CALL64m:
429       case X86::JMP64m:
430       case X86::TAILJMPm64:
431       case X86::TEST64mr:
432       case X86::ADC64rm:
433       case X86::ADD64rm:
434       case X86::AND64rm:
435       case X86::CMP64rm:
436       case X86::OR64rm:
437       case X86::SBB64rm:
438       case X86::SUB64rm:
439       case X86::XOR64rm:
440         return HasREX ? X86::reloc_riprel_4byte_relax_rex
441                       : X86::reloc_riprel_4byte_relax;
442       }
443     }();
444 
445     // rip-relative addressing is actually relative to the *next* instruction.
446     // Since an immediate can follow the mod/rm byte for an instruction, this
447     // means that we need to bias the displacement field of the instruction with
448     // the size of the immediate field. If we have this case, add it into the
449     // expression to emit.
450     // Note: rip-relative addressing using immediate displacement values should
451     // not be adjusted, assuming it was the user's intent.
452     int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
453                       ? X86II::getSizeOfImm(TSFlags)
454                       : 0;
455 
456     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
457                   Fixups, -ImmSize);
458     return;
459   }
460 
461   unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
462 
463   // 16-bit addressing forms of the ModR/M byte have a different encoding for
464   // the R/M field and are far more limited in which registers can be used.
465   if (is16BitMemOperand(MI, Op, STI)) {
466     if (BaseReg) {
467       // For 32-bit addressing, the row and column values in Table 2-2 are
468       // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
469       // some special cases. And getX86RegNum reflects that numbering.
470       // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
471       // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
472       // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
473       // while values 0-3 indicate the allowed combinations (base+index) of
474       // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
475       //
476       // R16Table[] is a lookup from the normal RegNo, to the row values from
477       // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
478       static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
479       unsigned RMfield = R16Table[BaseRegNo];
480 
481       assert(RMfield && "invalid 16-bit base register");
482 
483       if (IndexReg.getReg()) {
484         unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
485 
486         assert(IndexReg16 && "invalid 16-bit index register");
487         // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
488         assert(((IndexReg16 ^ RMfield) & 2) &&
489                "invalid 16-bit base/index register combination");
490         assert(Scale.getImm() == 1 &&
491                "invalid scale for 16-bit memory reference");
492 
493         // Allow base/index to appear in either order (although GAS doesn't).
494         if (IndexReg16 & 2)
495           RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
496         else
497           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
498       }
499 
500       if (Disp.isImm() && isInt<8>(Disp.getImm())) {
501         if (Disp.getImm() == 0 && RMfield != 6) {
502           // There is no displacement; just the register.
503           emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
504           return;
505         }
506         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
507         emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
508         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
509         return;
510       }
511       // This is the [REG]+disp16 case.
512       emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
513     } else {
514       assert(IndexReg.getReg() == 0 && "Unexpected index register!");
515       // There is no BaseReg; this is the plain [disp16] case.
516       emitByte(modRMByte(0, RegOpcodeField, 6), OS);
517     }
518 
519     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
520     emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
521     return;
522   }
523 
524   // Check for presence of {disp8} or {disp32} pseudo prefixes.
525   bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
526   bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
527 
528   // We only allow no displacement if no pseudo prefix is present.
529   bool AllowNoDisp = !UseDisp8 && !UseDisp32;
530   // Disp8 is allowed unless the {disp32} prefix is present.
531   bool AllowDisp8 = !UseDisp32;
532 
533   // Determine whether a SIB byte is needed.
534   if (// The SIB byte must be used if there is an index register or the
535       // encoding requires a SIB byte.
536       !ForceSIB && IndexReg.getReg() == 0 &&
537       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
538       // encode to an R/M value of 4, which indicates that a SIB byte is
539       // present.
540       BaseRegNo != N86::ESP &&
541       // If there is no base register and we're in 64-bit mode, we need a SIB
542       // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
543       (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
544 
545     if (BaseReg == 0) { // [disp32]     in X86-32 mode
546       emitByte(modRMByte(0, RegOpcodeField, 5), OS);
547       emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
548       return;
549     }
550 
551     // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
552     // simple indirect register encoding, this handles addresses like [EAX].
553     // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
554     // handle it by emitting a displacement of 0 later.
555     if (BaseRegNo != N86::EBP) {
556       if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
557         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
558         return;
559       }
560 
561       // If the displacement is @tlscall, treat it as a zero.
562       if (Disp.isExpr()) {
563         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
564         if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
565           // This is exclusively used by call *a@tlscall(base). The relocation
566           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
567           Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
568           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
569           return;
570         }
571       }
572     }
573 
574     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
575     // Including a compressed disp8 for EVEX instructions that support it.
576     // This also handles the 0 displacement for [EBP] or [R13]. We can't use
577     // disp8 if the {disp32} pseudo prefix is present.
578     if (Disp.isImm() && AllowDisp8) {
579       int ImmOffset = 0;
580       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
581         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
582         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
583                       ImmOffset);
584         return;
585       }
586     }
587 
588     // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
589     // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
590     // prevented using disp8 above.
591     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
592     unsigned Opcode = MI.getOpcode();
593     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
594                                                 : X86::reloc_signed_4byte;
595     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
596                   Fixups);
597     return;
598   }
599 
600   // We need a SIB byte, so start by outputting the ModR/M byte first
601   assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
602          "Cannot use ESP as index reg!");
603 
604   bool ForceDisp32 = false;
605   bool ForceDisp8 = false;
606   int ImmOffset = 0;
607   if (BaseReg == 0) {
608     // If there is no base register, we emit the special case SIB byte with
609     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
610     BaseRegNo = 5;
611     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
612     ForceDisp32 = true;
613   } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
614              // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
615              // the base field, but that is the magic [*] nomenclature that
616              // indicates no base when mod=0. For these cases we'll emit a 0
617              // displacement instead.
618              BaseRegNo != N86::EBP) {
619     // Emit no displacement ModR/M byte
620     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
621   } else if (Disp.isImm() && AllowDisp8 &&
622              isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
623     // Displacement fits in a byte or matches an EVEX compressed disp8, use
624     // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
625     // {disp32} pseudo prefix was used.
626     emitByte(modRMByte(1, RegOpcodeField, 4), OS);
627     ForceDisp8 = true;
628   } else {
629     // Otherwise, emit the normal disp32 encoding.
630     emitByte(modRMByte(2, RegOpcodeField, 4), OS);
631     ForceDisp32 = true;
632   }
633 
634   // Calculate what the SS field value should be...
635   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
636   unsigned SS = SSTable[Scale.getImm()];
637 
638   unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
639 
640   emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
641 
642   // Do we need to output a displacement?
643   if (ForceDisp8)
644     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
645                   ImmOffset);
646   else if (ForceDisp32)
647     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
648                   StartByte, OS, Fixups);
649 }
650 
651 /// Emit all instruction prefixes.
652 ///
653 /// \returns true if REX prefix is used, otherwise returns false.
654 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
655                                       const MCSubtargetInfo &STI,
656                                       raw_ostream &OS) const {
657   uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
658   // Determine where the memory operand starts, if present.
659   int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
660   // Emit segment override opcode prefix as needed.
661   if (MemoryOperand != -1) {
662     MemoryOperand += CurOp;
663     emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
664   }
665 
666   // Emit the repeat opcode prefix as needed.
667   unsigned Flags = MI.getFlags();
668   if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
669     emitByte(0xF3, OS);
670   if (Flags & X86::IP_HAS_REPEAT_NE)
671     emitByte(0xF2, OS);
672 
673   // Emit the address size opcode prefix as needed.
674   bool NeedAddressOverride;
675   uint64_t AdSize = TSFlags & X86II::AdSizeMask;
676   if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
677       (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
678       (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
679     NeedAddressOverride = true;
680   } else if (MemoryOperand < 0) {
681     NeedAddressOverride = false;
682   } else if (STI.hasFeature(X86::Mode64Bit)) {
683     assert(!is16BitMemOperand(MI, MemoryOperand, STI));
684     NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
685   } else if (STI.hasFeature(X86::Mode32Bit)) {
686     assert(!is64BitMemOperand(MI, MemoryOperand));
687     NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
688   } else {
689     assert(STI.hasFeature(X86::Mode16Bit));
690     assert(!is64BitMemOperand(MI, MemoryOperand));
691     NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
692   }
693 
694   if (NeedAddressOverride)
695     emitByte(0x67, OS);
696 
697   // Encoding type for this instruction.
698   uint64_t Encoding = TSFlags & X86II::EncodingMask;
699   bool HasREX = false;
700   if (Encoding)
701     emitVEXOpcodePrefix(MemoryOperand, MI, OS);
702   else
703     HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
704 
705   uint64_t Form = TSFlags & X86II::FormMask;
706   switch (Form) {
707   default:
708     break;
709   case X86II::RawFrmDstSrc: {
710     unsigned siReg = MI.getOperand(1).getReg();
711     assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
712             (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
713             (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
714            "SI and DI register sizes do not match");
715     // Emit segment override opcode prefix as needed (not for %ds).
716     if (MI.getOperand(2).getReg() != X86::DS)
717       emitSegmentOverridePrefix(2, MI, OS);
718     // Emit AdSize prefix as needed.
719     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
720         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
721       emitByte(0x67, OS);
722     CurOp += 3; // Consume operands.
723     break;
724   }
725   case X86II::RawFrmSrc: {
726     unsigned siReg = MI.getOperand(0).getReg();
727     // Emit segment override opcode prefix as needed (not for %ds).
728     if (MI.getOperand(1).getReg() != X86::DS)
729       emitSegmentOverridePrefix(1, MI, OS);
730     // Emit AdSize prefix as needed.
731     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
732         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
733       emitByte(0x67, OS);
734     CurOp += 2; // Consume operands.
735     break;
736   }
737   case X86II::RawFrmDst: {
738     unsigned siReg = MI.getOperand(0).getReg();
739     // Emit AdSize prefix as needed.
740     if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
741         (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
742       emitByte(0x67, OS);
743     ++CurOp; // Consume operand.
744     break;
745   }
746   case X86II::RawFrmMemOffs: {
747     // Emit segment override opcode prefix as needed.
748     emitSegmentOverridePrefix(1, MI, OS);
749     break;
750   }
751   }
752 
753   return HasREX;
754 }
755 
756 /// AVX instructions are encoded using a opcode prefix called VEX.
757 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
758                                            raw_ostream &OS) const {
759   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
760   uint64_t TSFlags = Desc.TSFlags;
761 
762   assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
763 
764   uint64_t Encoding = TSFlags & X86II::EncodingMask;
765   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
766   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
767   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
768 
769   // VEX_R: opcode externsion equivalent to REX.R in
770   // 1's complement (inverted) form
771   //
772   //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
773   //  0: Same as REX_R=1 (64 bit mode only)
774   //
775   uint8_t VEX_R = 0x1;
776   uint8_t EVEX_R2 = 0x1;
777 
778   // VEX_X: equivalent to REX.X, only used when a
779   // register is used for index in SIB Byte.
780   //
781   //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
782   //  0: Same as REX.X=1 (64-bit mode only)
783   uint8_t VEX_X = 0x1;
784 
785   // VEX_B:
786   //
787   //  1: Same as REX_B=0 (ignored in 32-bit mode)
788   //  0: Same as REX_B=1 (64 bit mode only)
789   //
790   uint8_t VEX_B = 0x1;
791 
792   // VEX_W: opcode specific (use like REX.W, or used for
793   // opcode extension, or ignored, depending on the opcode byte)
794   uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
795 
796   // VEX_5M (VEX m-mmmmm field):
797   //
798   //  0b00000: Reserved for future use
799   //  0b00001: implied 0F leading opcode
800   //  0b00010: implied 0F 38 leading opcode bytes
801   //  0b00011: implied 0F 3A leading opcode bytes
802   //  0b00100: Reserved for future use
803   //  0b00101: VEX MAP5
804   //  0b00110: VEX MAP6
805   //  0b00111-0b11111: Reserved for future use
806   //  0b01000: XOP map select - 08h instructions with imm byte
807   //  0b01001: XOP map select - 09h instructions with no imm byte
808   //  0b01010: XOP map select - 0Ah instructions with imm dword
809   uint8_t VEX_5M;
810   switch (TSFlags & X86II::OpMapMask) {
811   default:
812     llvm_unreachable("Invalid prefix!");
813   case X86II::TB:
814     VEX_5M = 0x1;
815     break; // 0F
816   case X86II::T8:
817     VEX_5M = 0x2;
818     break; // 0F 38
819   case X86II::TA:
820     VEX_5M = 0x3;
821     break; // 0F 3A
822   case X86II::XOP8:
823     VEX_5M = 0x8;
824     break;
825   case X86II::XOP9:
826     VEX_5M = 0x9;
827     break;
828   case X86II::XOPA:
829     VEX_5M = 0xA;
830     break;
831   case X86II::T_MAP5:
832     VEX_5M = 0x5;
833     break;
834   case X86II::T_MAP6:
835     VEX_5M = 0x6;
836     break;
837   }
838 
839   // VEX_4V (VEX vvvv field): a register specifier
840   // (in 1's complement form) or 1111 if unused.
841   uint8_t VEX_4V = 0xf;
842   uint8_t EVEX_V2 = 0x1;
843 
844   // EVEX_L2/VEX_L (Vector Length):
845   //
846   // L2 L
847   //  0 0: scalar or 128-bit vector
848   //  0 1: 256-bit vector
849   //  1 0: 512-bit vector
850   //
851   uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
852   uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
853 
854   // VEX_PP: opcode extension providing equivalent
855   // functionality of a SIMD prefix
856   //
857   //  0b00: None
858   //  0b01: 66
859   //  0b10: F3
860   //  0b11: F2
861   //
862   uint8_t VEX_PP = 0;
863   switch (TSFlags & X86II::OpPrefixMask) {
864   case X86II::PD:
865     VEX_PP = 0x1;
866     break; // 66
867   case X86II::XS:
868     VEX_PP = 0x2;
869     break; // F3
870   case X86II::XD:
871     VEX_PP = 0x3;
872     break; // F2
873   }
874 
875   // EVEX_U
876   uint8_t EVEX_U = 1; // Always '1' so far
877 
878   // EVEX_z
879   uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
880 
881   // EVEX_b
882   uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
883 
884   // EVEX_rc
885   uint8_t EVEX_rc = 0;
886 
887   // EVEX_aaa
888   uint8_t EVEX_aaa = 0;
889 
890   bool EncodeRC = false;
891 
892   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
893   unsigned NumOps = Desc.getNumOperands();
894   unsigned CurOp = X86II::getOperandBias(Desc);
895 
896   switch (TSFlags & X86II::FormMask) {
897   default:
898     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
899   case X86II::MRM_C0:
900   case X86II::RawFrm:
901   case X86II::PrefixByte:
902     break;
903   case X86II::MRMDestMemFSIB:
904   case X86II::MRMDestMem: {
905     // MRMDestMem instructions forms:
906     //  MemAddr, src1(ModR/M)
907     //  MemAddr, src1(VEX_4V), src2(ModR/M)
908     //  MemAddr, src1(ModR/M), imm8
909     //
910     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
911     VEX_B = ~(BaseRegEnc >> 3) & 1;
912     unsigned IndexRegEnc =
913         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
914     VEX_X = ~(IndexRegEnc >> 3) & 1;
915     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
916       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
917 
918     CurOp += X86::AddrNumOperands;
919 
920     if (HasEVEX_K)
921       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
922 
923     if (HasVEX_4V) {
924       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
925       VEX_4V = ~VRegEnc & 0xf;
926       EVEX_V2 = ~(VRegEnc >> 4) & 1;
927     }
928 
929     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
930     VEX_R = ~(RegEnc >> 3) & 1;
931     EVEX_R2 = ~(RegEnc >> 4) & 1;
932     break;
933   }
934   case X86II::MRMSrcMemFSIB:
935   case X86II::MRMSrcMem: {
936     // MRMSrcMem instructions forms:
937     //  src1(ModR/M), MemAddr
938     //  src1(ModR/M), src2(VEX_4V), MemAddr
939     //  src1(ModR/M), MemAddr, imm8
940     //  src1(ModR/M), MemAddr, src2(Imm[7:4])
941     //
942     //  FMA4:
943     //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
944     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
945     VEX_R = ~(RegEnc >> 3) & 1;
946     EVEX_R2 = ~(RegEnc >> 4) & 1;
947 
948     if (HasEVEX_K)
949       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
950 
951     if (HasVEX_4V) {
952       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
953       VEX_4V = ~VRegEnc & 0xf;
954       EVEX_V2 = ~(VRegEnc >> 4) & 1;
955     }
956 
957     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
958     VEX_B = ~(BaseRegEnc >> 3) & 1;
959     unsigned IndexRegEnc =
960         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
961     VEX_X = ~(IndexRegEnc >> 3) & 1;
962     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
963       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
964 
965     break;
966   }
967   case X86II::MRMSrcMem4VOp3: {
968     // Instruction format for 4VOp3:
969     //   src1(ModR/M), MemAddr, src3(VEX_4V)
970     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
971     VEX_R = ~(RegEnc >> 3) & 1;
972 
973     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
974     VEX_B = ~(BaseRegEnc >> 3) & 1;
975     unsigned IndexRegEnc =
976         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
977     VEX_X = ~(IndexRegEnc >> 3) & 1;
978 
979     VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
980     break;
981   }
982   case X86II::MRMSrcMemOp4: {
983     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
984     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
985     VEX_R = ~(RegEnc >> 3) & 1;
986 
987     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
988     VEX_4V = ~VRegEnc & 0xf;
989 
990     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
991     VEX_B = ~(BaseRegEnc >> 3) & 1;
992     unsigned IndexRegEnc =
993         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
994     VEX_X = ~(IndexRegEnc >> 3) & 1;
995     break;
996   }
997   case X86II::MRM0m:
998   case X86II::MRM1m:
999   case X86II::MRM2m:
1000   case X86II::MRM3m:
1001   case X86II::MRM4m:
1002   case X86II::MRM5m:
1003   case X86II::MRM6m:
1004   case X86II::MRM7m: {
1005     // MRM[0-9]m instructions forms:
1006     //  MemAddr
1007     //  src1(VEX_4V), MemAddr
1008     if (HasVEX_4V) {
1009       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1010       VEX_4V = ~VRegEnc & 0xf;
1011       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1012     }
1013 
1014     if (HasEVEX_K)
1015       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1016 
1017     unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
1018     VEX_B = ~(BaseRegEnc >> 3) & 1;
1019     unsigned IndexRegEnc =
1020         getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
1021     VEX_X = ~(IndexRegEnc >> 3) & 1;
1022     if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
1023       EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
1024 
1025     break;
1026   }
1027   case X86II::MRMSrcReg: {
1028     // MRMSrcReg instructions forms:
1029     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1030     //  dst(ModR/M), src1(ModR/M)
1031     //  dst(ModR/M), src1(ModR/M), imm8
1032     //
1033     //  FMA4:
1034     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1035     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1036     VEX_R = ~(RegEnc >> 3) & 1;
1037     EVEX_R2 = ~(RegEnc >> 4) & 1;
1038 
1039     if (HasEVEX_K)
1040       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1041 
1042     if (HasVEX_4V) {
1043       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1044       VEX_4V = ~VRegEnc & 0xf;
1045       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1046     }
1047 
1048     RegEnc = getX86RegEncoding(MI, CurOp++);
1049     VEX_B = ~(RegEnc >> 3) & 1;
1050     VEX_X = ~(RegEnc >> 4) & 1;
1051 
1052     if (EVEX_b) {
1053       if (HasEVEX_RC) {
1054         unsigned RcOperand = NumOps - 1;
1055         assert(RcOperand >= CurOp);
1056         EVEX_rc = MI.getOperand(RcOperand).getImm();
1057         assert(EVEX_rc <= 3 && "Invalid rounding control!");
1058       }
1059       EncodeRC = true;
1060     }
1061     break;
1062   }
1063   case X86II::MRMSrcReg4VOp3: {
1064     // Instruction format for 4VOp3:
1065     //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1066     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1067     VEX_R = ~(RegEnc >> 3) & 1;
1068 
1069     RegEnc = getX86RegEncoding(MI, CurOp++);
1070     VEX_B = ~(RegEnc >> 3) & 1;
1071 
1072     VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1073     break;
1074   }
1075   case X86II::MRMSrcRegOp4: {
1076     //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1077     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1078     VEX_R = ~(RegEnc >> 3) & 1;
1079 
1080     unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1081     VEX_4V = ~VRegEnc & 0xf;
1082 
1083     // Skip second register source (encoded in Imm[7:4])
1084     ++CurOp;
1085 
1086     RegEnc = getX86RegEncoding(MI, CurOp++);
1087     VEX_B = ~(RegEnc >> 3) & 1;
1088     VEX_X = ~(RegEnc >> 4) & 1;
1089     break;
1090   }
1091   case X86II::MRMDestReg: {
1092     // MRMDestReg instructions forms:
1093     //  dst(ModR/M), src(ModR/M)
1094     //  dst(ModR/M), src(ModR/M), imm8
1095     //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1096     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1097     VEX_B = ~(RegEnc >> 3) & 1;
1098     VEX_X = ~(RegEnc >> 4) & 1;
1099 
1100     if (HasEVEX_K)
1101       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1102 
1103     if (HasVEX_4V) {
1104       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1105       VEX_4V = ~VRegEnc & 0xf;
1106       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1107     }
1108 
1109     RegEnc = getX86RegEncoding(MI, CurOp++);
1110     VEX_R = ~(RegEnc >> 3) & 1;
1111     EVEX_R2 = ~(RegEnc >> 4) & 1;
1112     if (EVEX_b)
1113       EncodeRC = true;
1114     break;
1115   }
1116   case X86II::MRMr0: {
1117     // MRMr0 instructions forms:
1118     //  11:rrr:000
1119     //  dst(ModR/M)
1120     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1121     VEX_R = ~(RegEnc >> 3) & 1;
1122     EVEX_R2 = ~(RegEnc >> 4) & 1;
1123     break;
1124   }
1125   case X86II::MRM0r:
1126   case X86II::MRM1r:
1127   case X86II::MRM2r:
1128   case X86II::MRM3r:
1129   case X86II::MRM4r:
1130   case X86II::MRM5r:
1131   case X86II::MRM6r:
1132   case X86II::MRM7r: {
1133     // MRM0r-MRM7r instructions forms:
1134     //  dst(VEX_4V), src(ModR/M), imm8
1135     if (HasVEX_4V) {
1136       unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1137       VEX_4V = ~VRegEnc & 0xf;
1138       EVEX_V2 = ~(VRegEnc >> 4) & 1;
1139     }
1140     if (HasEVEX_K)
1141       EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1142 
1143     unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1144     VEX_B = ~(RegEnc >> 3) & 1;
1145     VEX_X = ~(RegEnc >> 4) & 1;
1146     break;
1147   }
1148   }
1149 
1150   if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1151     // VEX opcode prefix can have 2 or 3 bytes
1152     //
1153     //  3 bytes:
1154     //    +-----+ +--------------+ +-------------------+
1155     //    | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1156     //    +-----+ +--------------+ +-------------------+
1157     //  2 bytes:
1158     //    +-----+ +-------------------+
1159     //    | C5h | | R | vvvv | L | pp |
1160     //    +-----+ +-------------------+
1161     //
1162     //  XOP uses a similar prefix:
1163     //    +-----+ +--------------+ +-------------------+
1164     //    | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1165     //    +-----+ +--------------+ +-------------------+
1166     uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1167 
1168     // Can we use the 2 byte VEX prefix?
1169     if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1170         VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1171       emitByte(0xC5, OS);
1172       emitByte(LastByte | (VEX_R << 7), OS);
1173       return;
1174     }
1175 
1176     // 3 byte VEX prefix
1177     emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1178     emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1179     emitByte(LastByte | (VEX_W << 7), OS);
1180   } else {
1181     assert(Encoding == X86II::EVEX && "unknown encoding!");
1182     // EVEX opcode prefix can have 4 bytes
1183     //
1184     // +-----+ +--------------+ +-------------------+ +------------------------+
1185     // | 62h | | RXBR' | 0mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1186     // +-----+ +--------------+ +-------------------+ +------------------------+
1187     assert((VEX_5M & 0x7) == VEX_5M &&
1188            "More than 3 significant bits in VEX.m-mmmm fields for EVEX!");
1189 
1190     emitByte(0x62, OS);
1191     emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1192                  VEX_5M,
1193              OS);
1194     emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1195     if (EncodeRC)
1196       emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1197                    EVEX_aaa,
1198                OS);
1199     else
1200       emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1201                    (EVEX_V2 << 3) | EVEX_aaa,
1202                OS);
1203   }
1204 }
1205 
1206 /// Emit REX prefix which specifies
1207 ///   1) 64-bit instructions,
1208 ///   2) non-default operand size, and
1209 ///   3) use of X86-64 extended registers.
1210 ///
1211 /// \returns true if REX prefix is used, otherwise returns false.
1212 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1213                                      const MCSubtargetInfo &STI,
1214                                      raw_ostream &OS) const {
1215   uint8_t REX = [&, MemOperand]() {
1216     uint8_t REX = 0;
1217     bool UsesHighByteReg = false;
1218 
1219     const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1220     uint64_t TSFlags = Desc.TSFlags;
1221 
1222     if (TSFlags & X86II::REX_W)
1223       REX |= 1 << 3; // set REX.W
1224 
1225     if (MI.getNumOperands() == 0)
1226       return REX;
1227 
1228     unsigned NumOps = MI.getNumOperands();
1229     unsigned CurOp = X86II::getOperandBias(Desc);
1230 
1231     // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1232     for (unsigned i = CurOp; i != NumOps; ++i) {
1233       const MCOperand &MO = MI.getOperand(i);
1234       if (MO.isReg()) {
1235         unsigned Reg = MO.getReg();
1236         if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
1237             Reg == X86::DH)
1238           UsesHighByteReg = true;
1239         if (X86II::isX86_64NonExtLowByteReg(Reg))
1240           // FIXME: The caller of determineREXPrefix slaps this prefix onto
1241           // anything that returns non-zero.
1242           REX |= 0x40; // REX fixed encoding prefix
1243       } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {
1244         // GOTTPOFF and TLSDESC relocations require a REX prefix to allow
1245         // linker optimizations: even if the instructions we see may not require
1246         // any prefix, they may be replaced by instructions that do. This is
1247         // handled as a special case here so that it also works for hand-written
1248         // assembly without the user needing to write REX, as with GNU as.
1249         const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
1250         if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF ||
1251                     Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) {
1252           REX |= 0x40; // REX fixed encoding prefix
1253         }
1254       }
1255     }
1256 
1257     switch (TSFlags & X86II::FormMask) {
1258     case X86II::AddRegFrm:
1259       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1260       break;
1261     case X86II::MRMSrcReg:
1262     case X86II::MRMSrcRegCC:
1263       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1264       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1265       break;
1266     case X86II::MRMSrcMem:
1267     case X86II::MRMSrcMemCC:
1268       REX |= isREXExtendedReg(MI, CurOp++) << 2;                        // REX.R
1269       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1270       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1271       CurOp += X86::AddrNumOperands;
1272       break;
1273     case X86II::MRMDestReg:
1274       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1275       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1276       break;
1277     case X86II::MRMDestMem:
1278       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1279       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1280       CurOp += X86::AddrNumOperands;
1281       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1282       break;
1283     case X86II::MRMXmCC:
1284     case X86II::MRMXm:
1285     case X86II::MRM0m:
1286     case X86II::MRM1m:
1287     case X86II::MRM2m:
1288     case X86II::MRM3m:
1289     case X86II::MRM4m:
1290     case X86II::MRM5m:
1291     case X86II::MRM6m:
1292     case X86II::MRM7m:
1293       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0;  // REX.B
1294       REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1295       break;
1296     case X86II::MRMXrCC:
1297     case X86II::MRMXr:
1298     case X86II::MRM0r:
1299     case X86II::MRM1r:
1300     case X86II::MRM2r:
1301     case X86II::MRM3r:
1302     case X86II::MRM4r:
1303     case X86II::MRM5r:
1304     case X86II::MRM6r:
1305     case X86II::MRM7r:
1306       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1307       break;
1308     case X86II::MRMr0:
1309       REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1310       break;
1311     case X86II::MRMDestMemFSIB:
1312       llvm_unreachable("FSIB format never need REX prefix!");
1313     }
1314     if (REX && UsesHighByteReg)
1315       report_fatal_error(
1316           "Cannot encode high byte register in REX-prefixed instruction");
1317     return REX;
1318   }();
1319 
1320   if (!REX)
1321     return false;
1322 
1323   emitByte(0x40 | REX, OS);
1324   return true;
1325 }
1326 
1327 /// Emit segment override opcode prefix as needed.
1328 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1329                                                  const MCInst &MI,
1330                                                  raw_ostream &OS) const {
1331   // Check for explicit segment override on memory operand.
1332   if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1333     emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1334 }
1335 
1336 /// Emit all instruction prefixes prior to the opcode.
1337 ///
1338 /// \param MemOperand the operand # of the start of a memory operand if present.
1339 /// If not present, it is -1.
1340 ///
1341 /// \returns true if REX prefix is used, otherwise returns false.
1342 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1343                                         const MCSubtargetInfo &STI,
1344                                         raw_ostream &OS) const {
1345   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1346   uint64_t TSFlags = Desc.TSFlags;
1347 
1348   // Emit the operand size opcode prefix as needed.
1349   if ((TSFlags & X86II::OpSizeMask) ==
1350       (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1351     emitByte(0x66, OS);
1352 
1353   // Emit the LOCK opcode prefix.
1354   if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1355     emitByte(0xF0, OS);
1356 
1357   // Emit the NOTRACK opcode prefix.
1358   if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1359     emitByte(0x3E, OS);
1360 
1361   switch (TSFlags & X86II::OpPrefixMask) {
1362   case X86II::PD: // 66
1363     emitByte(0x66, OS);
1364     break;
1365   case X86II::XS: // F3
1366     emitByte(0xF3, OS);
1367     break;
1368   case X86II::XD: // F2
1369     emitByte(0xF2, OS);
1370     break;
1371   }
1372 
1373   // Handle REX prefix.
1374   assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1375          "REX.W requires 64bit mode.");
1376   bool HasREX = STI.hasFeature(X86::Mode64Bit)
1377                     ? emitREXPrefix(MemOperand, MI, STI, OS)
1378                     : false;
1379 
1380   // 0x0F escape code must be emitted just before the opcode.
1381   switch (TSFlags & X86II::OpMapMask) {
1382   case X86II::TB:        // Two-byte opcode map
1383   case X86II::T8:        // 0F 38
1384   case X86II::TA:        // 0F 3A
1385   case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1386     emitByte(0x0F, OS);
1387     break;
1388   }
1389 
1390   switch (TSFlags & X86II::OpMapMask) {
1391   case X86II::T8: // 0F 38
1392     emitByte(0x38, OS);
1393     break;
1394   case X86II::TA: // 0F 3A
1395     emitByte(0x3A, OS);
1396     break;
1397   }
1398 
1399   return HasREX;
1400 }
1401 
1402 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1403                                   const MCSubtargetInfo &STI) const {
1404   unsigned Opcode = MI.getOpcode();
1405   const MCInstrDesc &Desc = MCII.get(Opcode);
1406   uint64_t TSFlags = Desc.TSFlags;
1407 
1408   // Pseudo instructions don't get encoded.
1409   if (X86II::isPseudo(TSFlags))
1410     return;
1411 
1412   unsigned CurOp = X86II::getOperandBias(Desc);
1413 
1414   emitPrefixImpl(CurOp, MI, STI, OS);
1415 }
1416 
1417 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1418                                          SmallVectorImpl<MCFixup> &Fixups,
1419                                          const MCSubtargetInfo &STI) const {
1420   unsigned Opcode = MI.getOpcode();
1421   const MCInstrDesc &Desc = MCII.get(Opcode);
1422   uint64_t TSFlags = Desc.TSFlags;
1423 
1424   // Pseudo instructions don't get encoded.
1425   if (X86II::isPseudo(TSFlags))
1426     return;
1427 
1428   unsigned NumOps = Desc.getNumOperands();
1429   unsigned CurOp = X86II::getOperandBias(Desc);
1430 
1431   uint64_t StartByte = OS.tell();
1432 
1433   bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1434 
1435   // It uses the VEX.VVVV field?
1436   bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1437   bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1438 
1439   // It uses the EVEX.aaa field?
1440   bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1441   bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1442 
1443   // Used if a register is encoded in 7:4 of immediate.
1444   unsigned I8RegNum = 0;
1445 
1446   uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1447 
1448   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1449     BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1450 
1451   unsigned OpcodeOffset = 0;
1452 
1453   uint64_t Form = TSFlags & X86II::FormMask;
1454   switch (Form) {
1455   default:
1456     errs() << "FORM: " << Form << "\n";
1457     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1458   case X86II::Pseudo:
1459     llvm_unreachable("Pseudo instruction shouldn't be emitted");
1460   case X86II::RawFrmDstSrc:
1461   case X86II::RawFrmSrc:
1462   case X86II::RawFrmDst:
1463   case X86II::PrefixByte:
1464     emitByte(BaseOpcode, OS);
1465     break;
1466   case X86II::AddCCFrm: {
1467     // This will be added to the opcode in the fallthrough.
1468     OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1469     assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1470     --NumOps; // Drop the operand from the end.
1471     LLVM_FALLTHROUGH;
1472   case X86II::RawFrm:
1473     emitByte(BaseOpcode + OpcodeOffset, OS);
1474 
1475     if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1476       break;
1477 
1478     const MCOperand &Op = MI.getOperand(CurOp++);
1479     emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1480                   MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1481                   Fixups);
1482     break;
1483   }
1484   case X86II::RawFrmMemOffs:
1485     emitByte(BaseOpcode, OS);
1486     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1487                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1488                   StartByte, OS, Fixups);
1489     ++CurOp; // skip segment operand
1490     break;
1491   case X86II::RawFrmImm8:
1492     emitByte(BaseOpcode, OS);
1493     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1494                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1495                   StartByte, OS, Fixups);
1496     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1497                   OS, Fixups);
1498     break;
1499   case X86II::RawFrmImm16:
1500     emitByte(BaseOpcode, OS);
1501     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1502                   X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1503                   StartByte, OS, Fixups);
1504     emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1505                   OS, Fixups);
1506     break;
1507 
1508   case X86II::AddRegFrm:
1509     emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1510     break;
1511 
1512   case X86II::MRMDestReg: {
1513     emitByte(BaseOpcode, OS);
1514     unsigned SrcRegNum = CurOp + 1;
1515 
1516     if (HasEVEX_K) // Skip writemask
1517       ++SrcRegNum;
1518 
1519     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1520       ++SrcRegNum;
1521 
1522     emitRegModRMByte(MI.getOperand(CurOp),
1523                      getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1524     CurOp = SrcRegNum + 1;
1525     break;
1526   }
1527   case X86II::MRMDestMemFSIB:
1528   case X86II::MRMDestMem: {
1529     emitByte(BaseOpcode, OS);
1530     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1531 
1532     if (HasEVEX_K) // Skip writemask
1533       ++SrcRegNum;
1534 
1535     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1536       ++SrcRegNum;
1537 
1538     bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1539     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1540                      HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1541     CurOp = SrcRegNum + 1;
1542     break;
1543   }
1544   case X86II::MRMSrcReg: {
1545     emitByte(BaseOpcode, OS);
1546     unsigned SrcRegNum = CurOp + 1;
1547 
1548     if (HasEVEX_K) // Skip writemask
1549       ++SrcRegNum;
1550 
1551     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1552       ++SrcRegNum;
1553 
1554     emitRegModRMByte(MI.getOperand(SrcRegNum),
1555                      getX86RegNum(MI.getOperand(CurOp)), OS);
1556     CurOp = SrcRegNum + 1;
1557     if (HasVEX_I8Reg)
1558       I8RegNum = getX86RegEncoding(MI, CurOp++);
1559     // do not count the rounding control operand
1560     if (HasEVEX_RC)
1561       --NumOps;
1562     break;
1563   }
1564   case X86II::MRMSrcReg4VOp3: {
1565     emitByte(BaseOpcode, OS);
1566     unsigned SrcRegNum = CurOp + 1;
1567 
1568     emitRegModRMByte(MI.getOperand(SrcRegNum),
1569                      getX86RegNum(MI.getOperand(CurOp)), OS);
1570     CurOp = SrcRegNum + 1;
1571     ++CurOp; // Encoded in VEX.VVVV
1572     break;
1573   }
1574   case X86II::MRMSrcRegOp4: {
1575     emitByte(BaseOpcode, OS);
1576     unsigned SrcRegNum = CurOp + 1;
1577 
1578     // Skip 1st src (which is encoded in VEX_VVVV)
1579     ++SrcRegNum;
1580 
1581     // Capture 2nd src (which is encoded in Imm[7:4])
1582     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1583     I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1584 
1585     emitRegModRMByte(MI.getOperand(SrcRegNum),
1586                      getX86RegNum(MI.getOperand(CurOp)), OS);
1587     CurOp = SrcRegNum + 1;
1588     break;
1589   }
1590   case X86II::MRMSrcRegCC: {
1591     unsigned FirstOp = CurOp++;
1592     unsigned SecondOp = CurOp++;
1593 
1594     unsigned CC = MI.getOperand(CurOp++).getImm();
1595     emitByte(BaseOpcode + CC, OS);
1596 
1597     emitRegModRMByte(MI.getOperand(SecondOp),
1598                      getX86RegNum(MI.getOperand(FirstOp)), OS);
1599     break;
1600   }
1601   case X86II::MRMSrcMemFSIB:
1602   case X86II::MRMSrcMem: {
1603     unsigned FirstMemOp = CurOp + 1;
1604 
1605     if (HasEVEX_K) // Skip writemask
1606       ++FirstMemOp;
1607 
1608     if (HasVEX_4V)
1609       ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1610 
1611     emitByte(BaseOpcode, OS);
1612 
1613     bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1614     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1615                      TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1616     CurOp = FirstMemOp + X86::AddrNumOperands;
1617     if (HasVEX_I8Reg)
1618       I8RegNum = getX86RegEncoding(MI, CurOp++);
1619     break;
1620   }
1621   case X86II::MRMSrcMem4VOp3: {
1622     unsigned FirstMemOp = CurOp + 1;
1623 
1624     emitByte(BaseOpcode, OS);
1625 
1626     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1627                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1628     CurOp = FirstMemOp + X86::AddrNumOperands;
1629     ++CurOp; // Encoded in VEX.VVVV.
1630     break;
1631   }
1632   case X86II::MRMSrcMemOp4: {
1633     unsigned FirstMemOp = CurOp + 1;
1634 
1635     ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1636 
1637     // Capture second register source (encoded in Imm[7:4])
1638     assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1639     I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1640 
1641     emitByte(BaseOpcode, OS);
1642 
1643     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1644                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1645     CurOp = FirstMemOp + X86::AddrNumOperands;
1646     break;
1647   }
1648   case X86II::MRMSrcMemCC: {
1649     unsigned RegOp = CurOp++;
1650     unsigned FirstMemOp = CurOp;
1651     CurOp = FirstMemOp + X86::AddrNumOperands;
1652 
1653     unsigned CC = MI.getOperand(CurOp++).getImm();
1654     emitByte(BaseOpcode + CC, OS);
1655 
1656     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1657                      TSFlags, HasREX, StartByte, OS, Fixups, STI);
1658     break;
1659   }
1660 
1661   case X86II::MRMXrCC: {
1662     unsigned RegOp = CurOp++;
1663 
1664     unsigned CC = MI.getOperand(CurOp++).getImm();
1665     emitByte(BaseOpcode + CC, OS);
1666     emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1667     break;
1668   }
1669 
1670   case X86II::MRMXr:
1671   case X86II::MRM0r:
1672   case X86II::MRM1r:
1673   case X86II::MRM2r:
1674   case X86II::MRM3r:
1675   case X86II::MRM4r:
1676   case X86II::MRM5r:
1677   case X86II::MRM6r:
1678   case X86II::MRM7r:
1679     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1680       ++CurOp;
1681     if (HasEVEX_K) // Skip writemask
1682       ++CurOp;
1683     emitByte(BaseOpcode, OS);
1684     emitRegModRMByte(MI.getOperand(CurOp++),
1685                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1686     break;
1687   case X86II::MRMr0:
1688     emitByte(BaseOpcode, OS);
1689     emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1690     break;
1691 
1692   case X86II::MRMXmCC: {
1693     unsigned FirstMemOp = CurOp;
1694     CurOp = FirstMemOp + X86::AddrNumOperands;
1695 
1696     unsigned CC = MI.getOperand(CurOp++).getImm();
1697     emitByte(BaseOpcode + CC, OS);
1698 
1699     emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1700                      STI);
1701     break;
1702   }
1703 
1704   case X86II::MRMXm:
1705   case X86II::MRM0m:
1706   case X86II::MRM1m:
1707   case X86II::MRM2m:
1708   case X86II::MRM3m:
1709   case X86II::MRM4m:
1710   case X86II::MRM5m:
1711   case X86II::MRM6m:
1712   case X86II::MRM7m:
1713     if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1714       ++CurOp;
1715     if (HasEVEX_K) // Skip writemask
1716       ++CurOp;
1717     emitByte(BaseOpcode, OS);
1718     emitMemModRMByte(MI, CurOp,
1719                      (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1720                      HasREX, StartByte, OS, Fixups, STI);
1721     CurOp += X86::AddrNumOperands;
1722     break;
1723 
1724   case X86II::MRM0X:
1725   case X86II::MRM1X:
1726   case X86II::MRM2X:
1727   case X86II::MRM3X:
1728   case X86II::MRM4X:
1729   case X86II::MRM5X:
1730   case X86II::MRM6X:
1731   case X86II::MRM7X:
1732     emitByte(BaseOpcode, OS);
1733     emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1734     break;
1735 
1736   case X86II::MRM_C0:
1737   case X86II::MRM_C1:
1738   case X86II::MRM_C2:
1739   case X86II::MRM_C3:
1740   case X86II::MRM_C4:
1741   case X86II::MRM_C5:
1742   case X86II::MRM_C6:
1743   case X86II::MRM_C7:
1744   case X86II::MRM_C8:
1745   case X86II::MRM_C9:
1746   case X86II::MRM_CA:
1747   case X86II::MRM_CB:
1748   case X86II::MRM_CC:
1749   case X86II::MRM_CD:
1750   case X86II::MRM_CE:
1751   case X86II::MRM_CF:
1752   case X86II::MRM_D0:
1753   case X86II::MRM_D1:
1754   case X86II::MRM_D2:
1755   case X86II::MRM_D3:
1756   case X86II::MRM_D4:
1757   case X86II::MRM_D5:
1758   case X86II::MRM_D6:
1759   case X86II::MRM_D7:
1760   case X86II::MRM_D8:
1761   case X86II::MRM_D9:
1762   case X86II::MRM_DA:
1763   case X86II::MRM_DB:
1764   case X86II::MRM_DC:
1765   case X86II::MRM_DD:
1766   case X86II::MRM_DE:
1767   case X86II::MRM_DF:
1768   case X86II::MRM_E0:
1769   case X86II::MRM_E1:
1770   case X86II::MRM_E2:
1771   case X86II::MRM_E3:
1772   case X86II::MRM_E4:
1773   case X86II::MRM_E5:
1774   case X86II::MRM_E6:
1775   case X86II::MRM_E7:
1776   case X86II::MRM_E8:
1777   case X86II::MRM_E9:
1778   case X86II::MRM_EA:
1779   case X86II::MRM_EB:
1780   case X86II::MRM_EC:
1781   case X86II::MRM_ED:
1782   case X86II::MRM_EE:
1783   case X86II::MRM_EF:
1784   case X86II::MRM_F0:
1785   case X86II::MRM_F1:
1786   case X86II::MRM_F2:
1787   case X86II::MRM_F3:
1788   case X86II::MRM_F4:
1789   case X86II::MRM_F5:
1790   case X86II::MRM_F6:
1791   case X86II::MRM_F7:
1792   case X86II::MRM_F8:
1793   case X86II::MRM_F9:
1794   case X86II::MRM_FA:
1795   case X86II::MRM_FB:
1796   case X86II::MRM_FC:
1797   case X86II::MRM_FD:
1798   case X86II::MRM_FE:
1799   case X86II::MRM_FF:
1800     emitByte(BaseOpcode, OS);
1801     emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1802     break;
1803   }
1804 
1805   if (HasVEX_I8Reg) {
1806     // The last source register of a 4 operand instruction in AVX is encoded
1807     // in bits[7:4] of a immediate byte.
1808     assert(I8RegNum < 16 && "Register encoding out of range");
1809     I8RegNum <<= 4;
1810     if (CurOp != NumOps) {
1811       unsigned Val = MI.getOperand(CurOp++).getImm();
1812       assert(Val < 16 && "Immediate operand value out of range");
1813       I8RegNum |= Val;
1814     }
1815     emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1816                   StartByte, OS, Fixups);
1817   } else {
1818     // If there is a remaining operand, it must be a trailing immediate. Emit it
1819     // according to the right size for the instruction. Some instructions
1820     // (SSE4a extrq and insertq) have two trailing immediates.
1821     while (CurOp != NumOps && NumOps - CurOp <= 2) {
1822       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1823                     X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1824                     StartByte, OS, Fixups);
1825     }
1826   }
1827 
1828   if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1829     emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1830 
1831   assert(OS.tell() - StartByte <= 15 &&
1832          "The size of instruction must be no longer than 15.");
1833 #ifndef NDEBUG
1834   // FIXME: Verify.
1835   if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1836     errs() << "Cannot encode all operands of: ";
1837     MI.dump();
1838     errs() << '\n';
1839     abort();
1840   }
1841 #endif
1842 }
1843 
1844 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1845                                             const MCRegisterInfo &MRI,
1846                                             MCContext &Ctx) {
1847   return new X86MCCodeEmitter(MCII, Ctx);
1848 }
1849