1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32
33 using namespace llvm;
34
35 #define DEBUG_TYPE "mccodeemitter"
36
37 namespace {
38
39 class X86MCCodeEmitter : public MCCodeEmitter {
40 const MCInstrInfo &MCII;
41 MCContext &Ctx;
42
43 public:
X86MCCodeEmitter(const MCInstrInfo & mcii,MCContext & ctx)44 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45 : MCII(mcii), Ctx(ctx) {}
46 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48 ~X86MCCodeEmitter() override = default;
49
50 void emitPrefix(const MCInst &MI, raw_ostream &OS,
51 const MCSubtargetInfo &STI) const override;
52
53 void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54 SmallVectorImpl<MCFixup> &Fixups,
55 const MCSubtargetInfo &STI) const override;
56
57 private:
58 unsigned getX86RegNum(const MCOperand &MO) const;
59
60 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61
62 /// \param MI a single low-level machine instruction.
63 /// \param OpNum the operand #.
64 /// \returns true if the OpNumth operand of MI require a bit to be set in
65 /// REX prefix.
66 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67
68 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
70 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71
72 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73 raw_ostream &OS) const;
74
75 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76 raw_ostream &OS) const;
77
78 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79 uint64_t TSFlags, bool HasREX, uint64_t StartByte,
80 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81 const MCSubtargetInfo &STI,
82 bool ForceSIB = false) const;
83
84 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
85 const MCSubtargetInfo &STI, raw_ostream &OS) const;
86
87 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
88 raw_ostream &OS) const;
89
90 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
91 raw_ostream &OS) const;
92
93 bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
94 const MCSubtargetInfo &STI, raw_ostream &OS) const;
95
96 bool emitREXPrefix(int MemOperand, const MCInst &MI,
97 const MCSubtargetInfo &STI, raw_ostream &OS) const;
98 };
99
100 } // end anonymous namespace
101
modRMByte(unsigned Mod,unsigned RegOpcode,unsigned RM)102 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
103 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
104 return RM | (RegOpcode << 3) | (Mod << 6);
105 }
106
emitByte(uint8_t C,raw_ostream & OS)107 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
108
emitConstant(uint64_t Val,unsigned Size,raw_ostream & OS)109 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
110 // Output the constant in little endian byte order.
111 for (unsigned i = 0; i != Size; ++i) {
112 emitByte(Val & 255, OS);
113 Val >>= 8;
114 }
115 }
116
117 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
118 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
119 /// parameter of emitImmediate.
isDispOrCDisp8(uint64_t TSFlags,int Value,int & ImmOffset)120 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
121 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
122
123 int CD8_Scale =
124 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
125 if (!HasEVEX || CD8_Scale == 0)
126 return isInt<8>(Value);
127
128 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
129 if (Value & (CD8_Scale - 1)) // Unaligned offset
130 return false;
131
132 int CDisp8 = Value / CD8_Scale;
133 if (!isInt<8>(CDisp8))
134 return false;
135
136 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
137 ImmOffset = CDisp8 - Value;
138 return true;
139 }
140
141 /// \returns the appropriate fixup kind to use for an immediate in an
142 /// instruction with the specified TSFlags.
getImmFixupKind(uint64_t TSFlags)143 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
144 unsigned Size = X86II::getSizeOfImm(TSFlags);
145 bool isPCRel = X86II::isImmPCRel(TSFlags);
146
147 if (X86II::isImmSigned(TSFlags)) {
148 switch (Size) {
149 default:
150 llvm_unreachable("Unsupported signed fixup size!");
151 case 4:
152 return MCFixupKind(X86::reloc_signed_4byte);
153 }
154 }
155 return MCFixup::getKindForSize(Size, isPCRel);
156 }
157
158 /// \param Op operand # of the memory operand.
159 ///
160 /// \returns true if the specified instruction has a 16-bit memory operand.
is16BitMemOperand(const MCInst & MI,unsigned Op,const MCSubtargetInfo & STI)161 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
162 const MCSubtargetInfo &STI) {
163 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
164 const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg);
165
166 unsigned BaseReg = Base.getReg();
167 unsigned IndexReg = Index.getReg();
168
169 if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0)
170 return true;
171 if ((BaseReg != 0 &&
172 X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) ||
173 (IndexReg != 0 &&
174 X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)))
175 return true;
176 return false;
177 }
178
179 /// \param Op operand # of the memory operand.
180 ///
181 /// \returns true if the specified instruction has a 32-bit memory operand.
is32BitMemOperand(const MCInst & MI,unsigned Op)182 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
183 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
184 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
185
186 if ((BaseReg.getReg() != 0 &&
187 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
188 (IndexReg.getReg() != 0 &&
189 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
190 return true;
191 if (BaseReg.getReg() == X86::EIP) {
192 assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
193 return true;
194 }
195 if (IndexReg.getReg() == X86::EIZ)
196 return true;
197 return false;
198 }
199
200 /// \param Op operand # of the memory operand.
201 ///
202 /// \returns true if the specified instruction has a 64-bit memory operand.
203 #ifndef NDEBUG
is64BitMemOperand(const MCInst & MI,unsigned Op)204 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
205 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
206 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
207
208 if ((BaseReg.getReg() != 0 &&
209 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
210 (IndexReg.getReg() != 0 &&
211 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
212 return true;
213 return false;
214 }
215 #endif
216
217 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
218
219 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is
220 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
221 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
222 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
223 /// binary expression.
224 static GlobalOffsetTableExprKind
startsWithGlobalOffsetTable(const MCExpr * Expr)225 startsWithGlobalOffsetTable(const MCExpr *Expr) {
226 const MCExpr *RHS = nullptr;
227 if (Expr->getKind() == MCExpr::Binary) {
228 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
229 Expr = BE->getLHS();
230 RHS = BE->getRHS();
231 }
232
233 if (Expr->getKind() != MCExpr::SymbolRef)
234 return GOT_None;
235
236 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
237 const MCSymbol &S = Ref->getSymbol();
238 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
239 return GOT_None;
240 if (RHS && RHS->getKind() == MCExpr::SymbolRef)
241 return GOT_SymDiff;
242 return GOT_Normal;
243 }
244
hasSecRelSymbolRef(const MCExpr * Expr)245 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
246 if (Expr->getKind() == MCExpr::SymbolRef) {
247 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
248 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
249 }
250 return false;
251 }
252
isPCRel32Branch(const MCInst & MI,const MCInstrInfo & MCII)253 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
254 unsigned Opcode = MI.getOpcode();
255 const MCInstrDesc &Desc = MCII.get(Opcode);
256 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
257 Opcode != X86::JCC_4) ||
258 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
259 return false;
260
261 unsigned CurOp = X86II::getOperandBias(Desc);
262 const MCOperand &Op = MI.getOperand(CurOp);
263 if (!Op.isExpr())
264 return false;
265
266 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
267 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
268 }
269
getX86RegNum(const MCOperand & MO) const270 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
271 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
272 }
273
getX86RegEncoding(const MCInst & MI,unsigned OpNum) const274 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
275 unsigned OpNum) const {
276 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
277 }
278
279 /// \param MI a single low-level machine instruction.
280 /// \param OpNum the operand #.
281 /// \returns true if the OpNumth operand of MI require a bit to be set in
282 /// REX prefix.
isREXExtendedReg(const MCInst & MI,unsigned OpNum) const283 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
284 unsigned OpNum) const {
285 return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
286 }
287
emitImmediate(const MCOperand & DispOp,SMLoc Loc,unsigned Size,MCFixupKind FixupKind,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,int ImmOffset) const288 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
289 unsigned Size, MCFixupKind FixupKind,
290 uint64_t StartByte, raw_ostream &OS,
291 SmallVectorImpl<MCFixup> &Fixups,
292 int ImmOffset) const {
293 const MCExpr *Expr = nullptr;
294 if (DispOp.isImm()) {
295 // If this is a simple integer displacement that doesn't require a
296 // relocation, emit it now.
297 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
298 FixupKind != FK_PCRel_4) {
299 emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
300 return;
301 }
302 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
303 } else {
304 Expr = DispOp.getExpr();
305 }
306
307 // If we have an immoffset, add it to the expression.
308 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
309 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
310 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
311 if (Kind != GOT_None) {
312 assert(ImmOffset == 0);
313
314 if (Size == 8) {
315 FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
316 } else {
317 assert(Size == 4);
318 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
319 }
320
321 if (Kind == GOT_Normal)
322 ImmOffset = static_cast<int>(OS.tell() - StartByte);
323 } else if (Expr->getKind() == MCExpr::SymbolRef) {
324 if (hasSecRelSymbolRef(Expr)) {
325 FixupKind = MCFixupKind(FK_SecRel_4);
326 }
327 } else if (Expr->getKind() == MCExpr::Binary) {
328 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
329 if (hasSecRelSymbolRef(Bin->getLHS()) ||
330 hasSecRelSymbolRef(Bin->getRHS())) {
331 FixupKind = MCFixupKind(FK_SecRel_4);
332 }
333 }
334 }
335
336 // If the fixup is pc-relative, we need to bias the value to be relative to
337 // the start of the field, not the end of the field.
338 if (FixupKind == FK_PCRel_4 ||
339 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
340 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
341 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
342 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
343 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
344 ImmOffset -= 4;
345 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
346 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
347 // this needs to be a GOTPC32 relocation.
348 if (startsWithGlobalOffsetTable(Expr) != GOT_None)
349 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
350 }
351 if (FixupKind == FK_PCRel_2)
352 ImmOffset -= 2;
353 if (FixupKind == FK_PCRel_1)
354 ImmOffset -= 1;
355
356 if (ImmOffset)
357 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
358 Ctx);
359
360 // Emit a symbolic constant as a fixup and 4 zeros.
361 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
362 Expr, FixupKind, Loc));
363 emitConstant(0, Size, OS);
364 }
365
emitRegModRMByte(const MCOperand & ModRMReg,unsigned RegOpcodeFld,raw_ostream & OS) const366 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
367 unsigned RegOpcodeFld,
368 raw_ostream &OS) const {
369 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
370 }
371
emitSIBByte(unsigned SS,unsigned Index,unsigned Base,raw_ostream & OS) const372 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
373 raw_ostream &OS) const {
374 // SIB byte is in the same format as the modRMByte.
375 emitByte(modRMByte(SS, Index, Base), OS);
376 }
377
emitMemModRMByte(const MCInst & MI,unsigned Op,unsigned RegOpcodeField,uint64_t TSFlags,bool HasREX,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI,bool ForceSIB) const378 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
379 unsigned RegOpcodeField,
380 uint64_t TSFlags, bool HasREX,
381 uint64_t StartByte, raw_ostream &OS,
382 SmallVectorImpl<MCFixup> &Fixups,
383 const MCSubtargetInfo &STI,
384 bool ForceSIB) const {
385 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
386 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
387 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
388 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
389 unsigned BaseReg = Base.getReg();
390
391 // Handle %rip relative addressing.
392 if (BaseReg == X86::RIP ||
393 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
394 assert(STI.hasFeature(X86::Mode64Bit) &&
395 "Rip-relative addressing requires 64-bit mode");
396 assert(IndexReg.getReg() == 0 && !ForceSIB &&
397 "Invalid rip-relative address");
398 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
399
400 unsigned Opcode = MI.getOpcode();
401 unsigned FixupKind = [&]() {
402 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a
403 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
404 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))
405 return X86::reloc_riprel_4byte;
406
407 // Certain loads for GOT references can be relocated against the symbol
408 // directly if the symbol ends up in the same linkage unit.
409 switch (Opcode) {
410 default:
411 return X86::reloc_riprel_4byte;
412 case X86::MOV64rm:
413 // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a
414 // special case because COFF and Mach-O don't support ELF's more
415 // flexible R_X86_64_REX_GOTPCRELX relaxation.
416 assert(HasREX);
417 return X86::reloc_riprel_4byte_movq_load;
418 case X86::ADC32rm:
419 case X86::ADD32rm:
420 case X86::AND32rm:
421 case X86::CMP32rm:
422 case X86::MOV32rm:
423 case X86::OR32rm:
424 case X86::SBB32rm:
425 case X86::SUB32rm:
426 case X86::TEST32mr:
427 case X86::XOR32rm:
428 case X86::CALL64m:
429 case X86::JMP64m:
430 case X86::TAILJMPm64:
431 case X86::TEST64mr:
432 case X86::ADC64rm:
433 case X86::ADD64rm:
434 case X86::AND64rm:
435 case X86::CMP64rm:
436 case X86::OR64rm:
437 case X86::SBB64rm:
438 case X86::SUB64rm:
439 case X86::XOR64rm:
440 return HasREX ? X86::reloc_riprel_4byte_relax_rex
441 : X86::reloc_riprel_4byte_relax;
442 }
443 }();
444
445 // rip-relative addressing is actually relative to the *next* instruction.
446 // Since an immediate can follow the mod/rm byte for an instruction, this
447 // means that we need to bias the displacement field of the instruction with
448 // the size of the immediate field. If we have this case, add it into the
449 // expression to emit.
450 // Note: rip-relative addressing using immediate displacement values should
451 // not be adjusted, assuming it was the user's intent.
452 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
453 ? X86II::getSizeOfImm(TSFlags)
454 : 0;
455
456 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
457 Fixups, -ImmSize);
458 return;
459 }
460
461 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
462
463 // 16-bit addressing forms of the ModR/M byte have a different encoding for
464 // the R/M field and are far more limited in which registers can be used.
465 if (is16BitMemOperand(MI, Op, STI)) {
466 if (BaseReg) {
467 // For 32-bit addressing, the row and column values in Table 2-2 are
468 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
469 // some special cases. And getX86RegNum reflects that numbering.
470 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
471 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
472 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
473 // while values 0-3 indicate the allowed combinations (base+index) of
474 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
475 //
476 // R16Table[] is a lookup from the normal RegNo, to the row values from
477 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
478 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
479 unsigned RMfield = R16Table[BaseRegNo];
480
481 assert(RMfield && "invalid 16-bit base register");
482
483 if (IndexReg.getReg()) {
484 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
485
486 assert(IndexReg16 && "invalid 16-bit index register");
487 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
488 assert(((IndexReg16 ^ RMfield) & 2) &&
489 "invalid 16-bit base/index register combination");
490 assert(Scale.getImm() == 1 &&
491 "invalid scale for 16-bit memory reference");
492
493 // Allow base/index to appear in either order (although GAS doesn't).
494 if (IndexReg16 & 2)
495 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
496 else
497 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
498 }
499
500 if (Disp.isImm() && isInt<8>(Disp.getImm())) {
501 if (Disp.getImm() == 0 && RMfield != 6) {
502 // There is no displacement; just the register.
503 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
504 return;
505 }
506 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
507 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
508 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
509 return;
510 }
511 // This is the [REG]+disp16 case.
512 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
513 } else {
514 assert(IndexReg.getReg() == 0 && "Unexpected index register!");
515 // There is no BaseReg; this is the plain [disp16] case.
516 emitByte(modRMByte(0, RegOpcodeField, 6), OS);
517 }
518
519 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
520 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
521 return;
522 }
523
524 // Check for presence of {disp8} or {disp32} pseudo prefixes.
525 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
526 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
527
528 // We only allow no displacement if no pseudo prefix is present.
529 bool AllowNoDisp = !UseDisp8 && !UseDisp32;
530 // Disp8 is allowed unless the {disp32} prefix is present.
531 bool AllowDisp8 = !UseDisp32;
532
533 // Determine whether a SIB byte is needed.
534 if (// The SIB byte must be used if there is an index register or the
535 // encoding requires a SIB byte.
536 !ForceSIB && IndexReg.getReg() == 0 &&
537 // The SIB byte must be used if the base is ESP/RSP/R12, all of which
538 // encode to an R/M value of 4, which indicates that a SIB byte is
539 // present.
540 BaseRegNo != N86::ESP &&
541 // If there is no base register and we're in 64-bit mode, we need a SIB
542 // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
543 (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
544
545 if (BaseReg == 0) { // [disp32] in X86-32 mode
546 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
547 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
548 return;
549 }
550
551 // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
552 // simple indirect register encoding, this handles addresses like [EAX].
553 // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
554 // handle it by emitting a displacement of 0 later.
555 if (BaseRegNo != N86::EBP) {
556 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
557 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
558 return;
559 }
560
561 // If the displacement is @tlscall, treat it as a zero.
562 if (Disp.isExpr()) {
563 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
564 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
565 // This is exclusively used by call *a@tlscall(base). The relocation
566 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
567 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
568 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
569 return;
570 }
571 }
572 }
573
574 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
575 // Including a compressed disp8 for EVEX instructions that support it.
576 // This also handles the 0 displacement for [EBP] or [R13]. We can't use
577 // disp8 if the {disp32} pseudo prefix is present.
578 if (Disp.isImm() && AllowDisp8) {
579 int ImmOffset = 0;
580 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
581 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
582 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
583 ImmOffset);
584 return;
585 }
586 }
587
588 // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
589 // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
590 // prevented using disp8 above.
591 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
592 unsigned Opcode = MI.getOpcode();
593 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
594 : X86::reloc_signed_4byte;
595 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
596 Fixups);
597 return;
598 }
599
600 // We need a SIB byte, so start by outputting the ModR/M byte first
601 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
602 "Cannot use ESP as index reg!");
603
604 bool ForceDisp32 = false;
605 bool ForceDisp8 = false;
606 int ImmOffset = 0;
607 if (BaseReg == 0) {
608 // If there is no base register, we emit the special case SIB byte with
609 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
610 BaseRegNo = 5;
611 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
612 ForceDisp32 = true;
613 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
614 // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
615 // the base field, but that is the magic [*] nomenclature that
616 // indicates no base when mod=0. For these cases we'll emit a 0
617 // displacement instead.
618 BaseRegNo != N86::EBP) {
619 // Emit no displacement ModR/M byte
620 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
621 } else if (Disp.isImm() && AllowDisp8 &&
622 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
623 // Displacement fits in a byte or matches an EVEX compressed disp8, use
624 // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
625 // {disp32} pseudo prefix was used.
626 emitByte(modRMByte(1, RegOpcodeField, 4), OS);
627 ForceDisp8 = true;
628 } else {
629 // Otherwise, emit the normal disp32 encoding.
630 emitByte(modRMByte(2, RegOpcodeField, 4), OS);
631 ForceDisp32 = true;
632 }
633
634 // Calculate what the SS field value should be...
635 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
636 unsigned SS = SSTable[Scale.getImm()];
637
638 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
639
640 emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
641
642 // Do we need to output a displacement?
643 if (ForceDisp8)
644 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
645 ImmOffset);
646 else if (ForceDisp32)
647 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
648 StartByte, OS, Fixups);
649 }
650
651 /// Emit all instruction prefixes.
652 ///
653 /// \returns true if REX prefix is used, otherwise returns false.
emitPrefixImpl(unsigned & CurOp,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const654 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
655 const MCSubtargetInfo &STI,
656 raw_ostream &OS) const {
657 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
658 // Determine where the memory operand starts, if present.
659 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
660 // Emit segment override opcode prefix as needed.
661 if (MemoryOperand != -1) {
662 MemoryOperand += CurOp;
663 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
664 }
665
666 // Emit the repeat opcode prefix as needed.
667 unsigned Flags = MI.getFlags();
668 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
669 emitByte(0xF3, OS);
670 if (Flags & X86::IP_HAS_REPEAT_NE)
671 emitByte(0xF2, OS);
672
673 // Emit the address size opcode prefix as needed.
674 bool NeedAddressOverride;
675 uint64_t AdSize = TSFlags & X86II::AdSizeMask;
676 if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
677 (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
678 (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
679 NeedAddressOverride = true;
680 } else if (MemoryOperand < 0) {
681 NeedAddressOverride = false;
682 } else if (STI.hasFeature(X86::Mode64Bit)) {
683 assert(!is16BitMemOperand(MI, MemoryOperand, STI));
684 NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
685 } else if (STI.hasFeature(X86::Mode32Bit)) {
686 assert(!is64BitMemOperand(MI, MemoryOperand));
687 NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
688 } else {
689 assert(STI.hasFeature(X86::Mode16Bit));
690 assert(!is64BitMemOperand(MI, MemoryOperand));
691 NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
692 }
693
694 if (NeedAddressOverride)
695 emitByte(0x67, OS);
696
697 // Encoding type for this instruction.
698 uint64_t Encoding = TSFlags & X86II::EncodingMask;
699 bool HasREX = false;
700 if (Encoding)
701 emitVEXOpcodePrefix(MemoryOperand, MI, OS);
702 else
703 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
704
705 uint64_t Form = TSFlags & X86II::FormMask;
706 switch (Form) {
707 default:
708 break;
709 case X86II::RawFrmDstSrc: {
710 unsigned siReg = MI.getOperand(1).getReg();
711 assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
712 (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
713 (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
714 "SI and DI register sizes do not match");
715 // Emit segment override opcode prefix as needed (not for %ds).
716 if (MI.getOperand(2).getReg() != X86::DS)
717 emitSegmentOverridePrefix(2, MI, OS);
718 // Emit AdSize prefix as needed.
719 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
720 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
721 emitByte(0x67, OS);
722 CurOp += 3; // Consume operands.
723 break;
724 }
725 case X86II::RawFrmSrc: {
726 unsigned siReg = MI.getOperand(0).getReg();
727 // Emit segment override opcode prefix as needed (not for %ds).
728 if (MI.getOperand(1).getReg() != X86::DS)
729 emitSegmentOverridePrefix(1, MI, OS);
730 // Emit AdSize prefix as needed.
731 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
732 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
733 emitByte(0x67, OS);
734 CurOp += 2; // Consume operands.
735 break;
736 }
737 case X86II::RawFrmDst: {
738 unsigned siReg = MI.getOperand(0).getReg();
739 // Emit AdSize prefix as needed.
740 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
741 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
742 emitByte(0x67, OS);
743 ++CurOp; // Consume operand.
744 break;
745 }
746 case X86II::RawFrmMemOffs: {
747 // Emit segment override opcode prefix as needed.
748 emitSegmentOverridePrefix(1, MI, OS);
749 break;
750 }
751 }
752
753 return HasREX;
754 }
755
756 /// AVX instructions are encoded using a opcode prefix called VEX.
emitVEXOpcodePrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const757 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
758 raw_ostream &OS) const {
759 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
760 uint64_t TSFlags = Desc.TSFlags;
761
762 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
763
764 uint64_t Encoding = TSFlags & X86II::EncodingMask;
765 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
766 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
767 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
768
769 // VEX_R: opcode externsion equivalent to REX.R in
770 // 1's complement (inverted) form
771 //
772 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
773 // 0: Same as REX_R=1 (64 bit mode only)
774 //
775 uint8_t VEX_R = 0x1;
776 uint8_t EVEX_R2 = 0x1;
777
778 // VEX_X: equivalent to REX.X, only used when a
779 // register is used for index in SIB Byte.
780 //
781 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
782 // 0: Same as REX.X=1 (64-bit mode only)
783 uint8_t VEX_X = 0x1;
784
785 // VEX_B:
786 //
787 // 1: Same as REX_B=0 (ignored in 32-bit mode)
788 // 0: Same as REX_B=1 (64 bit mode only)
789 //
790 uint8_t VEX_B = 0x1;
791
792 // VEX_W: opcode specific (use like REX.W, or used for
793 // opcode extension, or ignored, depending on the opcode byte)
794 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
795
796 // VEX_5M (VEX m-mmmmm field):
797 //
798 // 0b00000: Reserved for future use
799 // 0b00001: implied 0F leading opcode
800 // 0b00010: implied 0F 38 leading opcode bytes
801 // 0b00011: implied 0F 3A leading opcode bytes
802 // 0b00100: Reserved for future use
803 // 0b00101: VEX MAP5
804 // 0b00110: VEX MAP6
805 // 0b00111-0b11111: Reserved for future use
806 // 0b01000: XOP map select - 08h instructions with imm byte
807 // 0b01001: XOP map select - 09h instructions with no imm byte
808 // 0b01010: XOP map select - 0Ah instructions with imm dword
809 uint8_t VEX_5M;
810 switch (TSFlags & X86II::OpMapMask) {
811 default:
812 llvm_unreachable("Invalid prefix!");
813 case X86II::TB:
814 VEX_5M = 0x1;
815 break; // 0F
816 case X86II::T8:
817 VEX_5M = 0x2;
818 break; // 0F 38
819 case X86II::TA:
820 VEX_5M = 0x3;
821 break; // 0F 3A
822 case X86II::XOP8:
823 VEX_5M = 0x8;
824 break;
825 case X86II::XOP9:
826 VEX_5M = 0x9;
827 break;
828 case X86II::XOPA:
829 VEX_5M = 0xA;
830 break;
831 case X86II::T_MAP5:
832 VEX_5M = 0x5;
833 break;
834 case X86II::T_MAP6:
835 VEX_5M = 0x6;
836 break;
837 }
838
839 // VEX_4V (VEX vvvv field): a register specifier
840 // (in 1's complement form) or 1111 if unused.
841 uint8_t VEX_4V = 0xf;
842 uint8_t EVEX_V2 = 0x1;
843
844 // EVEX_L2/VEX_L (Vector Length):
845 //
846 // L2 L
847 // 0 0: scalar or 128-bit vector
848 // 0 1: 256-bit vector
849 // 1 0: 512-bit vector
850 //
851 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
852 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
853
854 // VEX_PP: opcode extension providing equivalent
855 // functionality of a SIMD prefix
856 //
857 // 0b00: None
858 // 0b01: 66
859 // 0b10: F3
860 // 0b11: F2
861 //
862 uint8_t VEX_PP = 0;
863 switch (TSFlags & X86II::OpPrefixMask) {
864 case X86II::PD:
865 VEX_PP = 0x1;
866 break; // 66
867 case X86II::XS:
868 VEX_PP = 0x2;
869 break; // F3
870 case X86II::XD:
871 VEX_PP = 0x3;
872 break; // F2
873 }
874
875 // EVEX_U
876 uint8_t EVEX_U = 1; // Always '1' so far
877
878 // EVEX_z
879 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
880
881 // EVEX_b
882 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
883
884 // EVEX_rc
885 uint8_t EVEX_rc = 0;
886
887 // EVEX_aaa
888 uint8_t EVEX_aaa = 0;
889
890 bool EncodeRC = false;
891
892 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
893 unsigned NumOps = Desc.getNumOperands();
894 unsigned CurOp = X86II::getOperandBias(Desc);
895
896 switch (TSFlags & X86II::FormMask) {
897 default:
898 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
899 case X86II::MRM_C0:
900 case X86II::RawFrm:
901 case X86II::PrefixByte:
902 break;
903 case X86II::MRMDestMemFSIB:
904 case X86II::MRMDestMem: {
905 // MRMDestMem instructions forms:
906 // MemAddr, src1(ModR/M)
907 // MemAddr, src1(VEX_4V), src2(ModR/M)
908 // MemAddr, src1(ModR/M), imm8
909 //
910 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
911 VEX_B = ~(BaseRegEnc >> 3) & 1;
912 unsigned IndexRegEnc =
913 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
914 VEX_X = ~(IndexRegEnc >> 3) & 1;
915 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
916 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
917
918 CurOp += X86::AddrNumOperands;
919
920 if (HasEVEX_K)
921 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
922
923 if (HasVEX_4V) {
924 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
925 VEX_4V = ~VRegEnc & 0xf;
926 EVEX_V2 = ~(VRegEnc >> 4) & 1;
927 }
928
929 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
930 VEX_R = ~(RegEnc >> 3) & 1;
931 EVEX_R2 = ~(RegEnc >> 4) & 1;
932 break;
933 }
934 case X86II::MRMSrcMemFSIB:
935 case X86II::MRMSrcMem: {
936 // MRMSrcMem instructions forms:
937 // src1(ModR/M), MemAddr
938 // src1(ModR/M), src2(VEX_4V), MemAddr
939 // src1(ModR/M), MemAddr, imm8
940 // src1(ModR/M), MemAddr, src2(Imm[7:4])
941 //
942 // FMA4:
943 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
944 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
945 VEX_R = ~(RegEnc >> 3) & 1;
946 EVEX_R2 = ~(RegEnc >> 4) & 1;
947
948 if (HasEVEX_K)
949 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
950
951 if (HasVEX_4V) {
952 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
953 VEX_4V = ~VRegEnc & 0xf;
954 EVEX_V2 = ~(VRegEnc >> 4) & 1;
955 }
956
957 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
958 VEX_B = ~(BaseRegEnc >> 3) & 1;
959 unsigned IndexRegEnc =
960 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
961 VEX_X = ~(IndexRegEnc >> 3) & 1;
962 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
963 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
964
965 break;
966 }
967 case X86II::MRMSrcMem4VOp3: {
968 // Instruction format for 4VOp3:
969 // src1(ModR/M), MemAddr, src3(VEX_4V)
970 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
971 VEX_R = ~(RegEnc >> 3) & 1;
972
973 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
974 VEX_B = ~(BaseRegEnc >> 3) & 1;
975 unsigned IndexRegEnc =
976 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
977 VEX_X = ~(IndexRegEnc >> 3) & 1;
978
979 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
980 break;
981 }
982 case X86II::MRMSrcMemOp4: {
983 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
984 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
985 VEX_R = ~(RegEnc >> 3) & 1;
986
987 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
988 VEX_4V = ~VRegEnc & 0xf;
989
990 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
991 VEX_B = ~(BaseRegEnc >> 3) & 1;
992 unsigned IndexRegEnc =
993 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
994 VEX_X = ~(IndexRegEnc >> 3) & 1;
995 break;
996 }
997 case X86II::MRM0m:
998 case X86II::MRM1m:
999 case X86II::MRM2m:
1000 case X86II::MRM3m:
1001 case X86II::MRM4m:
1002 case X86II::MRM5m:
1003 case X86II::MRM6m:
1004 case X86II::MRM7m: {
1005 // MRM[0-9]m instructions forms:
1006 // MemAddr
1007 // src1(VEX_4V), MemAddr
1008 if (HasVEX_4V) {
1009 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1010 VEX_4V = ~VRegEnc & 0xf;
1011 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1012 }
1013
1014 if (HasEVEX_K)
1015 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1016
1017 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
1018 VEX_B = ~(BaseRegEnc >> 3) & 1;
1019 unsigned IndexRegEnc =
1020 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
1021 VEX_X = ~(IndexRegEnc >> 3) & 1;
1022 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
1023 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
1024
1025 break;
1026 }
1027 case X86II::MRMSrcReg: {
1028 // MRMSrcReg instructions forms:
1029 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1030 // dst(ModR/M), src1(ModR/M)
1031 // dst(ModR/M), src1(ModR/M), imm8
1032 //
1033 // FMA4:
1034 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1035 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1036 VEX_R = ~(RegEnc >> 3) & 1;
1037 EVEX_R2 = ~(RegEnc >> 4) & 1;
1038
1039 if (HasEVEX_K)
1040 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1041
1042 if (HasVEX_4V) {
1043 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1044 VEX_4V = ~VRegEnc & 0xf;
1045 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1046 }
1047
1048 RegEnc = getX86RegEncoding(MI, CurOp++);
1049 VEX_B = ~(RegEnc >> 3) & 1;
1050 VEX_X = ~(RegEnc >> 4) & 1;
1051
1052 if (EVEX_b) {
1053 if (HasEVEX_RC) {
1054 unsigned RcOperand = NumOps - 1;
1055 assert(RcOperand >= CurOp);
1056 EVEX_rc = MI.getOperand(RcOperand).getImm();
1057 assert(EVEX_rc <= 3 && "Invalid rounding control!");
1058 }
1059 EncodeRC = true;
1060 }
1061 break;
1062 }
1063 case X86II::MRMSrcReg4VOp3: {
1064 // Instruction format for 4VOp3:
1065 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1066 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1067 VEX_R = ~(RegEnc >> 3) & 1;
1068
1069 RegEnc = getX86RegEncoding(MI, CurOp++);
1070 VEX_B = ~(RegEnc >> 3) & 1;
1071
1072 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1073 break;
1074 }
1075 case X86II::MRMSrcRegOp4: {
1076 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1077 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1078 VEX_R = ~(RegEnc >> 3) & 1;
1079
1080 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1081 VEX_4V = ~VRegEnc & 0xf;
1082
1083 // Skip second register source (encoded in Imm[7:4])
1084 ++CurOp;
1085
1086 RegEnc = getX86RegEncoding(MI, CurOp++);
1087 VEX_B = ~(RegEnc >> 3) & 1;
1088 VEX_X = ~(RegEnc >> 4) & 1;
1089 break;
1090 }
1091 case X86II::MRMDestReg: {
1092 // MRMDestReg instructions forms:
1093 // dst(ModR/M), src(ModR/M)
1094 // dst(ModR/M), src(ModR/M), imm8
1095 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1096 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1097 VEX_B = ~(RegEnc >> 3) & 1;
1098 VEX_X = ~(RegEnc >> 4) & 1;
1099
1100 if (HasEVEX_K)
1101 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1102
1103 if (HasVEX_4V) {
1104 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1105 VEX_4V = ~VRegEnc & 0xf;
1106 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1107 }
1108
1109 RegEnc = getX86RegEncoding(MI, CurOp++);
1110 VEX_R = ~(RegEnc >> 3) & 1;
1111 EVEX_R2 = ~(RegEnc >> 4) & 1;
1112 if (EVEX_b)
1113 EncodeRC = true;
1114 break;
1115 }
1116 case X86II::MRMr0: {
1117 // MRMr0 instructions forms:
1118 // 11:rrr:000
1119 // dst(ModR/M)
1120 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1121 VEX_R = ~(RegEnc >> 3) & 1;
1122 EVEX_R2 = ~(RegEnc >> 4) & 1;
1123 break;
1124 }
1125 case X86II::MRM0r:
1126 case X86II::MRM1r:
1127 case X86II::MRM2r:
1128 case X86II::MRM3r:
1129 case X86II::MRM4r:
1130 case X86II::MRM5r:
1131 case X86II::MRM6r:
1132 case X86II::MRM7r: {
1133 // MRM0r-MRM7r instructions forms:
1134 // dst(VEX_4V), src(ModR/M), imm8
1135 if (HasVEX_4V) {
1136 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1137 VEX_4V = ~VRegEnc & 0xf;
1138 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1139 }
1140 if (HasEVEX_K)
1141 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1142
1143 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1144 VEX_B = ~(RegEnc >> 3) & 1;
1145 VEX_X = ~(RegEnc >> 4) & 1;
1146 break;
1147 }
1148 }
1149
1150 if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1151 // VEX opcode prefix can have 2 or 3 bytes
1152 //
1153 // 3 bytes:
1154 // +-----+ +--------------+ +-------------------+
1155 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1156 // +-----+ +--------------+ +-------------------+
1157 // 2 bytes:
1158 // +-----+ +-------------------+
1159 // | C5h | | R | vvvv | L | pp |
1160 // +-----+ +-------------------+
1161 //
1162 // XOP uses a similar prefix:
1163 // +-----+ +--------------+ +-------------------+
1164 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1165 // +-----+ +--------------+ +-------------------+
1166 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1167
1168 // Can we use the 2 byte VEX prefix?
1169 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1170 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1171 emitByte(0xC5, OS);
1172 emitByte(LastByte | (VEX_R << 7), OS);
1173 return;
1174 }
1175
1176 // 3 byte VEX prefix
1177 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1178 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1179 emitByte(LastByte | (VEX_W << 7), OS);
1180 } else {
1181 assert(Encoding == X86II::EVEX && "unknown encoding!");
1182 // EVEX opcode prefix can have 4 bytes
1183 //
1184 // +-----+ +--------------+ +-------------------+ +------------------------+
1185 // | 62h | | RXBR' | 0mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1186 // +-----+ +--------------+ +-------------------+ +------------------------+
1187 assert((VEX_5M & 0x7) == VEX_5M &&
1188 "More than 3 significant bits in VEX.m-mmmm fields for EVEX!");
1189
1190 emitByte(0x62, OS);
1191 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1192 VEX_5M,
1193 OS);
1194 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1195 if (EncodeRC)
1196 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1197 EVEX_aaa,
1198 OS);
1199 else
1200 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1201 (EVEX_V2 << 3) | EVEX_aaa,
1202 OS);
1203 }
1204 }
1205
1206 /// Emit REX prefix which specifies
1207 /// 1) 64-bit instructions,
1208 /// 2) non-default operand size, and
1209 /// 3) use of X86-64 extended registers.
1210 ///
1211 /// \returns true if REX prefix is used, otherwise returns false.
emitREXPrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1212 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1213 const MCSubtargetInfo &STI,
1214 raw_ostream &OS) const {
1215 uint8_t REX = [&, MemOperand]() {
1216 uint8_t REX = 0;
1217 bool UsesHighByteReg = false;
1218
1219 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1220 uint64_t TSFlags = Desc.TSFlags;
1221
1222 if (TSFlags & X86II::REX_W)
1223 REX |= 1 << 3; // set REX.W
1224
1225 if (MI.getNumOperands() == 0)
1226 return REX;
1227
1228 unsigned NumOps = MI.getNumOperands();
1229 unsigned CurOp = X86II::getOperandBias(Desc);
1230
1231 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1232 for (unsigned i = CurOp; i != NumOps; ++i) {
1233 const MCOperand &MO = MI.getOperand(i);
1234 if (MO.isReg()) {
1235 unsigned Reg = MO.getReg();
1236 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
1237 Reg == X86::DH)
1238 UsesHighByteReg = true;
1239 if (X86II::isX86_64NonExtLowByteReg(Reg))
1240 // FIXME: The caller of determineREXPrefix slaps this prefix onto
1241 // anything that returns non-zero.
1242 REX |= 0x40; // REX fixed encoding prefix
1243 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {
1244 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow
1245 // linker optimizations: even if the instructions we see may not require
1246 // any prefix, they may be replaced by instructions that do. This is
1247 // handled as a special case here so that it also works for hand-written
1248 // assembly without the user needing to write REX, as with GNU as.
1249 const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
1250 if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF ||
1251 Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) {
1252 REX |= 0x40; // REX fixed encoding prefix
1253 }
1254 }
1255 }
1256
1257 switch (TSFlags & X86II::FormMask) {
1258 case X86II::AddRegFrm:
1259 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1260 break;
1261 case X86II::MRMSrcReg:
1262 case X86II::MRMSrcRegCC:
1263 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1264 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1265 break;
1266 case X86II::MRMSrcMem:
1267 case X86II::MRMSrcMemCC:
1268 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1269 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1270 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1271 CurOp += X86::AddrNumOperands;
1272 break;
1273 case X86II::MRMDestReg:
1274 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1275 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1276 break;
1277 case X86II::MRMDestMem:
1278 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1279 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1280 CurOp += X86::AddrNumOperands;
1281 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1282 break;
1283 case X86II::MRMXmCC:
1284 case X86II::MRMXm:
1285 case X86II::MRM0m:
1286 case X86II::MRM1m:
1287 case X86II::MRM2m:
1288 case X86II::MRM3m:
1289 case X86II::MRM4m:
1290 case X86II::MRM5m:
1291 case X86II::MRM6m:
1292 case X86II::MRM7m:
1293 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1294 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1295 break;
1296 case X86II::MRMXrCC:
1297 case X86II::MRMXr:
1298 case X86II::MRM0r:
1299 case X86II::MRM1r:
1300 case X86II::MRM2r:
1301 case X86II::MRM3r:
1302 case X86II::MRM4r:
1303 case X86II::MRM5r:
1304 case X86II::MRM6r:
1305 case X86II::MRM7r:
1306 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1307 break;
1308 case X86II::MRMr0:
1309 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1310 break;
1311 case X86II::MRMDestMemFSIB:
1312 llvm_unreachable("FSIB format never need REX prefix!");
1313 }
1314 if (REX && UsesHighByteReg)
1315 report_fatal_error(
1316 "Cannot encode high byte register in REX-prefixed instruction");
1317 return REX;
1318 }();
1319
1320 if (!REX)
1321 return false;
1322
1323 emitByte(0x40 | REX, OS);
1324 return true;
1325 }
1326
1327 /// Emit segment override opcode prefix as needed.
emitSegmentOverridePrefix(unsigned SegOperand,const MCInst & MI,raw_ostream & OS) const1328 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1329 const MCInst &MI,
1330 raw_ostream &OS) const {
1331 // Check for explicit segment override on memory operand.
1332 if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1333 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1334 }
1335
1336 /// Emit all instruction prefixes prior to the opcode.
1337 ///
1338 /// \param MemOperand the operand # of the start of a memory operand if present.
1339 /// If not present, it is -1.
1340 ///
1341 /// \returns true if REX prefix is used, otherwise returns false.
emitOpcodePrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1342 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1343 const MCSubtargetInfo &STI,
1344 raw_ostream &OS) const {
1345 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1346 uint64_t TSFlags = Desc.TSFlags;
1347
1348 // Emit the operand size opcode prefix as needed.
1349 if ((TSFlags & X86II::OpSizeMask) ==
1350 (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1351 emitByte(0x66, OS);
1352
1353 // Emit the LOCK opcode prefix.
1354 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1355 emitByte(0xF0, OS);
1356
1357 // Emit the NOTRACK opcode prefix.
1358 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1359 emitByte(0x3E, OS);
1360
1361 switch (TSFlags & X86II::OpPrefixMask) {
1362 case X86II::PD: // 66
1363 emitByte(0x66, OS);
1364 break;
1365 case X86II::XS: // F3
1366 emitByte(0xF3, OS);
1367 break;
1368 case X86II::XD: // F2
1369 emitByte(0xF2, OS);
1370 break;
1371 }
1372
1373 // Handle REX prefix.
1374 assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1375 "REX.W requires 64bit mode.");
1376 bool HasREX = STI.hasFeature(X86::Mode64Bit)
1377 ? emitREXPrefix(MemOperand, MI, STI, OS)
1378 : false;
1379
1380 // 0x0F escape code must be emitted just before the opcode.
1381 switch (TSFlags & X86II::OpMapMask) {
1382 case X86II::TB: // Two-byte opcode map
1383 case X86II::T8: // 0F 38
1384 case X86II::TA: // 0F 3A
1385 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1386 emitByte(0x0F, OS);
1387 break;
1388 }
1389
1390 switch (TSFlags & X86II::OpMapMask) {
1391 case X86II::T8: // 0F 38
1392 emitByte(0x38, OS);
1393 break;
1394 case X86II::TA: // 0F 3A
1395 emitByte(0x3A, OS);
1396 break;
1397 }
1398
1399 return HasREX;
1400 }
1401
emitPrefix(const MCInst & MI,raw_ostream & OS,const MCSubtargetInfo & STI) const1402 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1403 const MCSubtargetInfo &STI) const {
1404 unsigned Opcode = MI.getOpcode();
1405 const MCInstrDesc &Desc = MCII.get(Opcode);
1406 uint64_t TSFlags = Desc.TSFlags;
1407
1408 // Pseudo instructions don't get encoded.
1409 if (X86II::isPseudo(TSFlags))
1410 return;
1411
1412 unsigned CurOp = X86II::getOperandBias(Desc);
1413
1414 emitPrefixImpl(CurOp, MI, STI, OS);
1415 }
1416
encodeInstruction(const MCInst & MI,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI) const1417 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1418 SmallVectorImpl<MCFixup> &Fixups,
1419 const MCSubtargetInfo &STI) const {
1420 unsigned Opcode = MI.getOpcode();
1421 const MCInstrDesc &Desc = MCII.get(Opcode);
1422 uint64_t TSFlags = Desc.TSFlags;
1423
1424 // Pseudo instructions don't get encoded.
1425 if (X86II::isPseudo(TSFlags))
1426 return;
1427
1428 unsigned NumOps = Desc.getNumOperands();
1429 unsigned CurOp = X86II::getOperandBias(Desc);
1430
1431 uint64_t StartByte = OS.tell();
1432
1433 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1434
1435 // It uses the VEX.VVVV field?
1436 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1437 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1438
1439 // It uses the EVEX.aaa field?
1440 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1441 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1442
1443 // Used if a register is encoded in 7:4 of immediate.
1444 unsigned I8RegNum = 0;
1445
1446 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1447
1448 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1449 BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1450
1451 unsigned OpcodeOffset = 0;
1452
1453 uint64_t Form = TSFlags & X86II::FormMask;
1454 switch (Form) {
1455 default:
1456 errs() << "FORM: " << Form << "\n";
1457 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1458 case X86II::Pseudo:
1459 llvm_unreachable("Pseudo instruction shouldn't be emitted");
1460 case X86II::RawFrmDstSrc:
1461 case X86II::RawFrmSrc:
1462 case X86II::RawFrmDst:
1463 case X86II::PrefixByte:
1464 emitByte(BaseOpcode, OS);
1465 break;
1466 case X86II::AddCCFrm: {
1467 // This will be added to the opcode in the fallthrough.
1468 OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1469 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1470 --NumOps; // Drop the operand from the end.
1471 LLVM_FALLTHROUGH;
1472 case X86II::RawFrm:
1473 emitByte(BaseOpcode + OpcodeOffset, OS);
1474
1475 if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1476 break;
1477
1478 const MCOperand &Op = MI.getOperand(CurOp++);
1479 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1480 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1481 Fixups);
1482 break;
1483 }
1484 case X86II::RawFrmMemOffs:
1485 emitByte(BaseOpcode, OS);
1486 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1487 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1488 StartByte, OS, Fixups);
1489 ++CurOp; // skip segment operand
1490 break;
1491 case X86II::RawFrmImm8:
1492 emitByte(BaseOpcode, OS);
1493 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1494 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1495 StartByte, OS, Fixups);
1496 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1497 OS, Fixups);
1498 break;
1499 case X86II::RawFrmImm16:
1500 emitByte(BaseOpcode, OS);
1501 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1502 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1503 StartByte, OS, Fixups);
1504 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1505 OS, Fixups);
1506 break;
1507
1508 case X86II::AddRegFrm:
1509 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1510 break;
1511
1512 case X86II::MRMDestReg: {
1513 emitByte(BaseOpcode, OS);
1514 unsigned SrcRegNum = CurOp + 1;
1515
1516 if (HasEVEX_K) // Skip writemask
1517 ++SrcRegNum;
1518
1519 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1520 ++SrcRegNum;
1521
1522 emitRegModRMByte(MI.getOperand(CurOp),
1523 getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1524 CurOp = SrcRegNum + 1;
1525 break;
1526 }
1527 case X86II::MRMDestMemFSIB:
1528 case X86II::MRMDestMem: {
1529 emitByte(BaseOpcode, OS);
1530 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1531
1532 if (HasEVEX_K) // Skip writemask
1533 ++SrcRegNum;
1534
1535 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1536 ++SrcRegNum;
1537
1538 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1539 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1540 HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1541 CurOp = SrcRegNum + 1;
1542 break;
1543 }
1544 case X86II::MRMSrcReg: {
1545 emitByte(BaseOpcode, OS);
1546 unsigned SrcRegNum = CurOp + 1;
1547
1548 if (HasEVEX_K) // Skip writemask
1549 ++SrcRegNum;
1550
1551 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1552 ++SrcRegNum;
1553
1554 emitRegModRMByte(MI.getOperand(SrcRegNum),
1555 getX86RegNum(MI.getOperand(CurOp)), OS);
1556 CurOp = SrcRegNum + 1;
1557 if (HasVEX_I8Reg)
1558 I8RegNum = getX86RegEncoding(MI, CurOp++);
1559 // do not count the rounding control operand
1560 if (HasEVEX_RC)
1561 --NumOps;
1562 break;
1563 }
1564 case X86II::MRMSrcReg4VOp3: {
1565 emitByte(BaseOpcode, OS);
1566 unsigned SrcRegNum = CurOp + 1;
1567
1568 emitRegModRMByte(MI.getOperand(SrcRegNum),
1569 getX86RegNum(MI.getOperand(CurOp)), OS);
1570 CurOp = SrcRegNum + 1;
1571 ++CurOp; // Encoded in VEX.VVVV
1572 break;
1573 }
1574 case X86II::MRMSrcRegOp4: {
1575 emitByte(BaseOpcode, OS);
1576 unsigned SrcRegNum = CurOp + 1;
1577
1578 // Skip 1st src (which is encoded in VEX_VVVV)
1579 ++SrcRegNum;
1580
1581 // Capture 2nd src (which is encoded in Imm[7:4])
1582 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1583 I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1584
1585 emitRegModRMByte(MI.getOperand(SrcRegNum),
1586 getX86RegNum(MI.getOperand(CurOp)), OS);
1587 CurOp = SrcRegNum + 1;
1588 break;
1589 }
1590 case X86II::MRMSrcRegCC: {
1591 unsigned FirstOp = CurOp++;
1592 unsigned SecondOp = CurOp++;
1593
1594 unsigned CC = MI.getOperand(CurOp++).getImm();
1595 emitByte(BaseOpcode + CC, OS);
1596
1597 emitRegModRMByte(MI.getOperand(SecondOp),
1598 getX86RegNum(MI.getOperand(FirstOp)), OS);
1599 break;
1600 }
1601 case X86II::MRMSrcMemFSIB:
1602 case X86II::MRMSrcMem: {
1603 unsigned FirstMemOp = CurOp + 1;
1604
1605 if (HasEVEX_K) // Skip writemask
1606 ++FirstMemOp;
1607
1608 if (HasVEX_4V)
1609 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1610
1611 emitByte(BaseOpcode, OS);
1612
1613 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1614 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1615 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1616 CurOp = FirstMemOp + X86::AddrNumOperands;
1617 if (HasVEX_I8Reg)
1618 I8RegNum = getX86RegEncoding(MI, CurOp++);
1619 break;
1620 }
1621 case X86II::MRMSrcMem4VOp3: {
1622 unsigned FirstMemOp = CurOp + 1;
1623
1624 emitByte(BaseOpcode, OS);
1625
1626 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1627 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1628 CurOp = FirstMemOp + X86::AddrNumOperands;
1629 ++CurOp; // Encoded in VEX.VVVV.
1630 break;
1631 }
1632 case X86II::MRMSrcMemOp4: {
1633 unsigned FirstMemOp = CurOp + 1;
1634
1635 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1636
1637 // Capture second register source (encoded in Imm[7:4])
1638 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1639 I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1640
1641 emitByte(BaseOpcode, OS);
1642
1643 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1644 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1645 CurOp = FirstMemOp + X86::AddrNumOperands;
1646 break;
1647 }
1648 case X86II::MRMSrcMemCC: {
1649 unsigned RegOp = CurOp++;
1650 unsigned FirstMemOp = CurOp;
1651 CurOp = FirstMemOp + X86::AddrNumOperands;
1652
1653 unsigned CC = MI.getOperand(CurOp++).getImm();
1654 emitByte(BaseOpcode + CC, OS);
1655
1656 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1657 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1658 break;
1659 }
1660
1661 case X86II::MRMXrCC: {
1662 unsigned RegOp = CurOp++;
1663
1664 unsigned CC = MI.getOperand(CurOp++).getImm();
1665 emitByte(BaseOpcode + CC, OS);
1666 emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1667 break;
1668 }
1669
1670 case X86II::MRMXr:
1671 case X86II::MRM0r:
1672 case X86II::MRM1r:
1673 case X86II::MRM2r:
1674 case X86II::MRM3r:
1675 case X86II::MRM4r:
1676 case X86II::MRM5r:
1677 case X86II::MRM6r:
1678 case X86II::MRM7r:
1679 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1680 ++CurOp;
1681 if (HasEVEX_K) // Skip writemask
1682 ++CurOp;
1683 emitByte(BaseOpcode, OS);
1684 emitRegModRMByte(MI.getOperand(CurOp++),
1685 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1686 break;
1687 case X86II::MRMr0:
1688 emitByte(BaseOpcode, OS);
1689 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1690 break;
1691
1692 case X86II::MRMXmCC: {
1693 unsigned FirstMemOp = CurOp;
1694 CurOp = FirstMemOp + X86::AddrNumOperands;
1695
1696 unsigned CC = MI.getOperand(CurOp++).getImm();
1697 emitByte(BaseOpcode + CC, OS);
1698
1699 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1700 STI);
1701 break;
1702 }
1703
1704 case X86II::MRMXm:
1705 case X86II::MRM0m:
1706 case X86II::MRM1m:
1707 case X86II::MRM2m:
1708 case X86II::MRM3m:
1709 case X86II::MRM4m:
1710 case X86II::MRM5m:
1711 case X86II::MRM6m:
1712 case X86II::MRM7m:
1713 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1714 ++CurOp;
1715 if (HasEVEX_K) // Skip writemask
1716 ++CurOp;
1717 emitByte(BaseOpcode, OS);
1718 emitMemModRMByte(MI, CurOp,
1719 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1720 HasREX, StartByte, OS, Fixups, STI);
1721 CurOp += X86::AddrNumOperands;
1722 break;
1723
1724 case X86II::MRM0X:
1725 case X86II::MRM1X:
1726 case X86II::MRM2X:
1727 case X86II::MRM3X:
1728 case X86II::MRM4X:
1729 case X86II::MRM5X:
1730 case X86II::MRM6X:
1731 case X86II::MRM7X:
1732 emitByte(BaseOpcode, OS);
1733 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1734 break;
1735
1736 case X86II::MRM_C0:
1737 case X86II::MRM_C1:
1738 case X86II::MRM_C2:
1739 case X86II::MRM_C3:
1740 case X86II::MRM_C4:
1741 case X86II::MRM_C5:
1742 case X86II::MRM_C6:
1743 case X86II::MRM_C7:
1744 case X86II::MRM_C8:
1745 case X86II::MRM_C9:
1746 case X86II::MRM_CA:
1747 case X86II::MRM_CB:
1748 case X86II::MRM_CC:
1749 case X86II::MRM_CD:
1750 case X86II::MRM_CE:
1751 case X86II::MRM_CF:
1752 case X86II::MRM_D0:
1753 case X86II::MRM_D1:
1754 case X86II::MRM_D2:
1755 case X86II::MRM_D3:
1756 case X86II::MRM_D4:
1757 case X86II::MRM_D5:
1758 case X86II::MRM_D6:
1759 case X86II::MRM_D7:
1760 case X86II::MRM_D8:
1761 case X86II::MRM_D9:
1762 case X86II::MRM_DA:
1763 case X86II::MRM_DB:
1764 case X86II::MRM_DC:
1765 case X86II::MRM_DD:
1766 case X86II::MRM_DE:
1767 case X86II::MRM_DF:
1768 case X86II::MRM_E0:
1769 case X86II::MRM_E1:
1770 case X86II::MRM_E2:
1771 case X86II::MRM_E3:
1772 case X86II::MRM_E4:
1773 case X86II::MRM_E5:
1774 case X86II::MRM_E6:
1775 case X86II::MRM_E7:
1776 case X86II::MRM_E8:
1777 case X86II::MRM_E9:
1778 case X86II::MRM_EA:
1779 case X86II::MRM_EB:
1780 case X86II::MRM_EC:
1781 case X86II::MRM_ED:
1782 case X86II::MRM_EE:
1783 case X86II::MRM_EF:
1784 case X86II::MRM_F0:
1785 case X86II::MRM_F1:
1786 case X86II::MRM_F2:
1787 case X86II::MRM_F3:
1788 case X86II::MRM_F4:
1789 case X86II::MRM_F5:
1790 case X86II::MRM_F6:
1791 case X86II::MRM_F7:
1792 case X86II::MRM_F8:
1793 case X86II::MRM_F9:
1794 case X86II::MRM_FA:
1795 case X86II::MRM_FB:
1796 case X86II::MRM_FC:
1797 case X86II::MRM_FD:
1798 case X86II::MRM_FE:
1799 case X86II::MRM_FF:
1800 emitByte(BaseOpcode, OS);
1801 emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1802 break;
1803 }
1804
1805 if (HasVEX_I8Reg) {
1806 // The last source register of a 4 operand instruction in AVX is encoded
1807 // in bits[7:4] of a immediate byte.
1808 assert(I8RegNum < 16 && "Register encoding out of range");
1809 I8RegNum <<= 4;
1810 if (CurOp != NumOps) {
1811 unsigned Val = MI.getOperand(CurOp++).getImm();
1812 assert(Val < 16 && "Immediate operand value out of range");
1813 I8RegNum |= Val;
1814 }
1815 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1816 StartByte, OS, Fixups);
1817 } else {
1818 // If there is a remaining operand, it must be a trailing immediate. Emit it
1819 // according to the right size for the instruction. Some instructions
1820 // (SSE4a extrq and insertq) have two trailing immediates.
1821 while (CurOp != NumOps && NumOps - CurOp <= 2) {
1822 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1823 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1824 StartByte, OS, Fixups);
1825 }
1826 }
1827
1828 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1829 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1830
1831 assert(OS.tell() - StartByte <= 15 &&
1832 "The size of instruction must be no longer than 15.");
1833 #ifndef NDEBUG
1834 // FIXME: Verify.
1835 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1836 errs() << "Cannot encode all operands of: ";
1837 MI.dump();
1838 errs() << '\n';
1839 abort();
1840 }
1841 #endif
1842 }
1843
createX86MCCodeEmitter(const MCInstrInfo & MCII,const MCRegisterInfo & MRI,MCContext & Ctx)1844 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1845 const MCRegisterInfo &MRI,
1846 MCContext &Ctx) {
1847 return new X86MCCodeEmitter(MCII, Ctx);
1848 }
1849