1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32
33 using namespace llvm;
34
35 #define DEBUG_TYPE "mccodeemitter"
36
37 namespace {
38
39 class X86MCCodeEmitter : public MCCodeEmitter {
40 const MCInstrInfo &MCII;
41 MCContext &Ctx;
42
43 public:
X86MCCodeEmitter(const MCInstrInfo & mcii,MCContext & ctx)44 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45 : MCII(mcii), Ctx(ctx) {}
46 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48 ~X86MCCodeEmitter() override = default;
49
50 void emitPrefix(const MCInst &MI, raw_ostream &OS,
51 const MCSubtargetInfo &STI) const override;
52
53 void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54 SmallVectorImpl<MCFixup> &Fixups,
55 const MCSubtargetInfo &STI) const override;
56
57 private:
58 unsigned getX86RegNum(const MCOperand &MO) const;
59
60 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61
62 /// \param MI a single low-level machine instruction.
63 /// \param OpNum the operand #.
64 /// \returns true if the OpNumth operand of MI require a bit to be set in
65 /// REX prefix.
66 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67
68 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
70 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71
72 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73 raw_ostream &OS) const;
74
75 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76 raw_ostream &OS) const;
77
78 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79 uint64_t TSFlags, bool HasREX, uint64_t StartByte,
80 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81 const MCSubtargetInfo &STI,
82 bool ForceSIB = false) const;
83
84 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
85 const MCSubtargetInfo &STI, raw_ostream &OS) const;
86
87 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
88 raw_ostream &OS) const;
89
90 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
91 raw_ostream &OS) const;
92
93 bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
94 const MCSubtargetInfo &STI, raw_ostream &OS) const;
95
96 bool emitREXPrefix(int MemOperand, const MCInst &MI, raw_ostream &OS) const;
97 };
98
99 } // end anonymous namespace
100
modRMByte(unsigned Mod,unsigned RegOpcode,unsigned RM)101 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
102 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
103 return RM | (RegOpcode << 3) | (Mod << 6);
104 }
105
emitByte(uint8_t C,raw_ostream & OS)106 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
107
emitConstant(uint64_t Val,unsigned Size,raw_ostream & OS)108 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
109 // Output the constant in little endian byte order.
110 for (unsigned i = 0; i != Size; ++i) {
111 emitByte(Val & 255, OS);
112 Val >>= 8;
113 }
114 }
115
116 /// \returns true if this signed displacement fits in a 8-bit sign-extended
117 /// field.
isDisp8(int Value)118 static bool isDisp8(int Value) { return Value == (int8_t)Value; }
119
120 /// \returns true if this signed displacement fits in a 8-bit compressed
121 /// dispacement field.
isCDisp8(uint64_t TSFlags,int Value,int & CValue)122 static bool isCDisp8(uint64_t TSFlags, int Value, int &CValue) {
123 assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
124 "Compressed 8-bit displacement is only valid for EVEX inst.");
125
126 unsigned CD8_Scale =
127 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
128 if (CD8_Scale == 0) {
129 CValue = Value;
130 return isDisp8(Value);
131 }
132
133 unsigned Mask = CD8_Scale - 1;
134 assert((CD8_Scale & Mask) == 0 && "Invalid memory object size.");
135 if (Value & Mask) // Unaligned offset
136 return false;
137 Value /= (int)CD8_Scale;
138 bool Ret = (Value == (int8_t)Value);
139
140 if (Ret)
141 CValue = Value;
142 return Ret;
143 }
144
145 /// \returns the appropriate fixup kind to use for an immediate in an
146 /// instruction with the specified TSFlags.
getImmFixupKind(uint64_t TSFlags)147 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
148 unsigned Size = X86II::getSizeOfImm(TSFlags);
149 bool isPCRel = X86II::isImmPCRel(TSFlags);
150
151 if (X86II::isImmSigned(TSFlags)) {
152 switch (Size) {
153 default:
154 llvm_unreachable("Unsupported signed fixup size!");
155 case 4:
156 return MCFixupKind(X86::reloc_signed_4byte);
157 }
158 }
159 return MCFixup::getKindForSize(Size, isPCRel);
160 }
161
162 /// \param Op operand # of the memory operand.
163 ///
164 /// \returns true if the specified instruction has a 16-bit memory operand.
is16BitMemOperand(const MCInst & MI,unsigned Op,const MCSubtargetInfo & STI)165 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
166 const MCSubtargetInfo &STI) {
167 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
168 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
169 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
170
171 if (STI.hasFeature(X86::Mode16Bit) && BaseReg.getReg() == 0 && Disp.isImm() &&
172 Disp.getImm() < 0x10000)
173 return true;
174 if ((BaseReg.getReg() != 0 &&
175 X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
176 (IndexReg.getReg() != 0 &&
177 X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
178 return true;
179 return false;
180 }
181
182 /// \param Op operand # of the memory operand.
183 ///
184 /// \returns true if the specified instruction has a 32-bit memory operand.
is32BitMemOperand(const MCInst & MI,unsigned Op)185 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
186 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
187 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
188
189 if ((BaseReg.getReg() != 0 &&
190 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
191 (IndexReg.getReg() != 0 &&
192 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
193 return true;
194 if (BaseReg.getReg() == X86::EIP) {
195 assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
196 return true;
197 }
198 if (IndexReg.getReg() == X86::EIZ)
199 return true;
200 return false;
201 }
202
203 /// \param Op operand # of the memory operand.
204 ///
205 /// \returns true if the specified instruction has a 64-bit memory operand.
206 #ifndef NDEBUG
is64BitMemOperand(const MCInst & MI,unsigned Op)207 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
208 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
209 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
210
211 if ((BaseReg.getReg() != 0 &&
212 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
213 (IndexReg.getReg() != 0 &&
214 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
215 return true;
216 return false;
217 }
218 #endif
219
220 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
221
222 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is
223 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
224 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
225 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
226 /// binary expression.
227 static GlobalOffsetTableExprKind
startsWithGlobalOffsetTable(const MCExpr * Expr)228 startsWithGlobalOffsetTable(const MCExpr *Expr) {
229 const MCExpr *RHS = nullptr;
230 if (Expr->getKind() == MCExpr::Binary) {
231 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
232 Expr = BE->getLHS();
233 RHS = BE->getRHS();
234 }
235
236 if (Expr->getKind() != MCExpr::SymbolRef)
237 return GOT_None;
238
239 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
240 const MCSymbol &S = Ref->getSymbol();
241 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
242 return GOT_None;
243 if (RHS && RHS->getKind() == MCExpr::SymbolRef)
244 return GOT_SymDiff;
245 return GOT_Normal;
246 }
247
hasSecRelSymbolRef(const MCExpr * Expr)248 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
249 if (Expr->getKind() == MCExpr::SymbolRef) {
250 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
251 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
252 }
253 return false;
254 }
255
isPCRel32Branch(const MCInst & MI,const MCInstrInfo & MCII)256 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
257 unsigned Opcode = MI.getOpcode();
258 const MCInstrDesc &Desc = MCII.get(Opcode);
259 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
260 Opcode != X86::JCC_4) ||
261 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
262 return false;
263
264 unsigned CurOp = X86II::getOperandBias(Desc);
265 const MCOperand &Op = MI.getOperand(CurOp);
266 if (!Op.isExpr())
267 return false;
268
269 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
270 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
271 }
272
getX86RegNum(const MCOperand & MO) const273 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
274 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
275 }
276
getX86RegEncoding(const MCInst & MI,unsigned OpNum) const277 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
278 unsigned OpNum) const {
279 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
280 }
281
282 /// \param MI a single low-level machine instruction.
283 /// \param OpNum the operand #.
284 /// \returns true if the OpNumth operand of MI require a bit to be set in
285 /// REX prefix.
isREXExtendedReg(const MCInst & MI,unsigned OpNum) const286 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
287 unsigned OpNum) const {
288 return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
289 }
290
emitImmediate(const MCOperand & DispOp,SMLoc Loc,unsigned Size,MCFixupKind FixupKind,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,int ImmOffset) const291 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
292 unsigned Size, MCFixupKind FixupKind,
293 uint64_t StartByte, raw_ostream &OS,
294 SmallVectorImpl<MCFixup> &Fixups,
295 int ImmOffset) const {
296 const MCExpr *Expr = nullptr;
297 if (DispOp.isImm()) {
298 // If this is a simple integer displacement that doesn't require a
299 // relocation, emit it now.
300 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
301 FixupKind != FK_PCRel_4) {
302 emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
303 return;
304 }
305 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
306 } else {
307 Expr = DispOp.getExpr();
308 }
309
310 // If we have an immoffset, add it to the expression.
311 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
312 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
313 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
314 if (Kind != GOT_None) {
315 assert(ImmOffset == 0);
316
317 if (Size == 8) {
318 FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
319 } else {
320 assert(Size == 4);
321 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
322 }
323
324 if (Kind == GOT_Normal)
325 ImmOffset = static_cast<int>(OS.tell() - StartByte);
326 } else if (Expr->getKind() == MCExpr::SymbolRef) {
327 if (hasSecRelSymbolRef(Expr)) {
328 FixupKind = MCFixupKind(FK_SecRel_4);
329 }
330 } else if (Expr->getKind() == MCExpr::Binary) {
331 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
332 if (hasSecRelSymbolRef(Bin->getLHS()) ||
333 hasSecRelSymbolRef(Bin->getRHS())) {
334 FixupKind = MCFixupKind(FK_SecRel_4);
335 }
336 }
337 }
338
339 // If the fixup is pc-relative, we need to bias the value to be relative to
340 // the start of the field, not the end of the field.
341 if (FixupKind == FK_PCRel_4 ||
342 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
343 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
344 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
345 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
346 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
347 ImmOffset -= 4;
348 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
349 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
350 // this needs to be a GOTPC32 relocation.
351 if (startsWithGlobalOffsetTable(Expr) != GOT_None)
352 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
353 }
354 if (FixupKind == FK_PCRel_2)
355 ImmOffset -= 2;
356 if (FixupKind == FK_PCRel_1)
357 ImmOffset -= 1;
358
359 if (ImmOffset)
360 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
361 Ctx);
362
363 // Emit a symbolic constant as a fixup and 4 zeros.
364 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
365 Expr, FixupKind, Loc));
366 emitConstant(0, Size, OS);
367 }
368
emitRegModRMByte(const MCOperand & ModRMReg,unsigned RegOpcodeFld,raw_ostream & OS) const369 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
370 unsigned RegOpcodeFld,
371 raw_ostream &OS) const {
372 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
373 }
374
emitSIBByte(unsigned SS,unsigned Index,unsigned Base,raw_ostream & OS) const375 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
376 raw_ostream &OS) const {
377 // SIB byte is in the same format as the modRMByte.
378 emitByte(modRMByte(SS, Index, Base), OS);
379 }
380
emitMemModRMByte(const MCInst & MI,unsigned Op,unsigned RegOpcodeField,uint64_t TSFlags,bool HasREX,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI,bool ForceSIB) const381 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
382 unsigned RegOpcodeField,
383 uint64_t TSFlags, bool HasREX,
384 uint64_t StartByte, raw_ostream &OS,
385 SmallVectorImpl<MCFixup> &Fixups,
386 const MCSubtargetInfo &STI,
387 bool ForceSIB) const {
388 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
389 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
390 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
391 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
392 unsigned BaseReg = Base.getReg();
393 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
394
395 // Handle %rip relative addressing.
396 if (BaseReg == X86::RIP ||
397 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
398 assert(STI.hasFeature(X86::Mode64Bit) &&
399 "Rip-relative addressing requires 64-bit mode");
400 assert(IndexReg.getReg() == 0 && !ForceSIB &&
401 "Invalid rip-relative address");
402 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
403
404 unsigned Opcode = MI.getOpcode();
405 // movq loads are handled with a special relocation form which allows the
406 // linker to eliminate some loads for GOT references which end up in the
407 // same linkage unit.
408 unsigned FixupKind = [=]() {
409 switch (Opcode) {
410 default:
411 return X86::reloc_riprel_4byte;
412 case X86::MOV64rm:
413 assert(HasREX);
414 return X86::reloc_riprel_4byte_movq_load;
415 case X86::CALL64m:
416 case X86::JMP64m:
417 case X86::TAILJMPm64:
418 case X86::TEST64mr:
419 case X86::ADC64rm:
420 case X86::ADD64rm:
421 case X86::AND64rm:
422 case X86::CMP64rm:
423 case X86::OR64rm:
424 case X86::SBB64rm:
425 case X86::SUB64rm:
426 case X86::XOR64rm:
427 return HasREX ? X86::reloc_riprel_4byte_relax_rex
428 : X86::reloc_riprel_4byte_relax;
429 }
430 }();
431
432 // rip-relative addressing is actually relative to the *next* instruction.
433 // Since an immediate can follow the mod/rm byte for an instruction, this
434 // means that we need to bias the displacement field of the instruction with
435 // the size of the immediate field. If we have this case, add it into the
436 // expression to emit.
437 // Note: rip-relative addressing using immediate displacement values should
438 // not be adjusted, assuming it was the user's intent.
439 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
440 ? X86II::getSizeOfImm(TSFlags)
441 : 0;
442
443 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
444 Fixups, -ImmSize);
445 return;
446 }
447
448 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
449
450 // 16-bit addressing forms of the ModR/M byte have a different encoding for
451 // the R/M field and are far more limited in which registers can be used.
452 if (is16BitMemOperand(MI, Op, STI)) {
453 if (BaseReg) {
454 // For 32-bit addressing, the row and column values in Table 2-2 are
455 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
456 // some special cases. And getX86RegNum reflects that numbering.
457 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
458 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
459 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
460 // while values 0-3 indicate the allowed combinations (base+index) of
461 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
462 //
463 // R16Table[] is a lookup from the normal RegNo, to the row values from
464 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
465 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
466 unsigned RMfield = R16Table[BaseRegNo];
467
468 assert(RMfield && "invalid 16-bit base register");
469
470 if (IndexReg.getReg()) {
471 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
472
473 assert(IndexReg16 && "invalid 16-bit index register");
474 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
475 assert(((IndexReg16 ^ RMfield) & 2) &&
476 "invalid 16-bit base/index register combination");
477 assert(Scale.getImm() == 1 &&
478 "invalid scale for 16-bit memory reference");
479
480 // Allow base/index to appear in either order (although GAS doesn't).
481 if (IndexReg16 & 2)
482 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
483 else
484 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
485 }
486
487 if (Disp.isImm() && isDisp8(Disp.getImm())) {
488 if (Disp.getImm() == 0 && RMfield != 6) {
489 // There is no displacement; just the register.
490 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
491 return;
492 }
493 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
494 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
495 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
496 return;
497 }
498 // This is the [REG]+disp16 case.
499 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
500 } else {
501 // There is no BaseReg; this is the plain [disp16] case.
502 emitByte(modRMByte(0, RegOpcodeField, 6), OS);
503 }
504
505 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
506 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
507 return;
508 }
509
510 // Determine whether a SIB byte is needed.
511 // If no BaseReg, issue a RIP relative instruction only if the MCE can
512 // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
513 // 2-7) and absolute references.
514
515 if ( // The SIB byte must be used if there is an index register.
516 !ForceSIB && IndexReg.getReg() == 0 &&
517 // The SIB byte must be used if the base is ESP/RSP/R12, all of which
518 // encode to an R/M value of 4, which indicates that a SIB byte is
519 // present.
520 BaseRegNo != N86::ESP &&
521 // If there is no base register and we're in 64-bit mode, we need a SIB
522 // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
523 (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
524
525 if (BaseReg == 0) { // [disp32] in X86-32 mode
526 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
527 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
528 return;
529 }
530
531 // If the base is not EBP/ESP and there is no displacement, use simple
532 // indirect register encoding, this handles addresses like [EAX]. The
533 // encoding for [EBP] with no displacement means [disp32] so we handle it
534 // by emitting a displacement of 0 below.
535 if (BaseRegNo != N86::EBP) {
536 if (Disp.isImm() && Disp.getImm() == 0) {
537 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
538 return;
539 }
540
541 // If the displacement is @tlscall, treat it as a zero.
542 if (Disp.isExpr()) {
543 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
544 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
545 // This is exclusively used by call *a@tlscall(base). The relocation
546 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
547 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
548 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
549 return;
550 }
551 }
552 }
553
554 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
555 if (Disp.isImm()) {
556 if (!HasEVEX && isDisp8(Disp.getImm())) {
557 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
558 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
559 return;
560 }
561 // Try EVEX compressed 8-bit displacement first; if failed, fall back to
562 // 32-bit displacement.
563 int CDisp8 = 0;
564 if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
565 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
566 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
567 CDisp8 - Disp.getImm());
568 return;
569 }
570 }
571
572 // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
573 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
574 unsigned Opcode = MI.getOpcode();
575 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
576 : X86::reloc_signed_4byte;
577 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
578 Fixups);
579 return;
580 }
581
582 // We need a SIB byte, so start by outputting the ModR/M byte first
583 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
584 "Cannot use ESP as index reg!");
585
586 bool ForceDisp32 = false;
587 bool ForceDisp8 = false;
588 int CDisp8 = 0;
589 int ImmOffset = 0;
590 if (BaseReg == 0) {
591 // If there is no base register, we emit the special case SIB byte with
592 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
593 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
594 ForceDisp32 = true;
595 } else if (!Disp.isImm()) {
596 // Emit the normal disp32 encoding.
597 emitByte(modRMByte(2, RegOpcodeField, 4), OS);
598 ForceDisp32 = true;
599 } else if (Disp.getImm() == 0 &&
600 // Base reg can't be anything that ends up with '5' as the base
601 // reg, it is the magic [*] nomenclature that indicates no base.
602 BaseRegNo != N86::EBP) {
603 // Emit no displacement ModR/M byte
604 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
605 } else if (!HasEVEX && isDisp8(Disp.getImm())) {
606 // Emit the disp8 encoding.
607 emitByte(modRMByte(1, RegOpcodeField, 4), OS);
608 ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
609 } else if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
610 // Emit the disp8 encoding.
611 emitByte(modRMByte(1, RegOpcodeField, 4), OS);
612 ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
613 ImmOffset = CDisp8 - Disp.getImm();
614 } else {
615 // Emit the normal disp32 encoding.
616 emitByte(modRMByte(2, RegOpcodeField, 4), OS);
617 }
618
619 // Calculate what the SS field value should be...
620 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
621 unsigned SS = SSTable[Scale.getImm()];
622
623 if (BaseReg == 0) {
624 // Handle the SIB byte for the case where there is no base, see Intel
625 // Manual 2A, table 2-7. The displacement has already been output.
626 unsigned IndexRegNo;
627 if (IndexReg.getReg())
628 IndexRegNo = getX86RegNum(IndexReg);
629 else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
630 IndexRegNo = 4;
631 emitSIBByte(SS, IndexRegNo, 5, OS);
632 } else {
633 unsigned IndexRegNo;
634 if (IndexReg.getReg())
635 IndexRegNo = getX86RegNum(IndexReg);
636 else
637 IndexRegNo = 4; // For example [ESP+1*<noreg>+4]
638 emitSIBByte(SS, IndexRegNo, getX86RegNum(Base), OS);
639 }
640
641 // Do we need to output a displacement?
642 if (ForceDisp8)
643 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
644 ImmOffset);
645 else if (ForceDisp32 || Disp.getImm() != 0)
646 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
647 StartByte, OS, Fixups);
648 }
649
650 /// Emit all instruction prefixes.
651 ///
652 /// \returns true if REX prefix is used, otherwise returns false.
emitPrefixImpl(unsigned & CurOp,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const653 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
654 const MCSubtargetInfo &STI,
655 raw_ostream &OS) const {
656 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
657 // Determine where the memory operand starts, if present.
658 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
659 // Emit segment override opcode prefix as needed.
660 if (MemoryOperand != -1) {
661 MemoryOperand += CurOp;
662 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
663 }
664
665 // Emit the repeat opcode prefix as needed.
666 unsigned Flags = MI.getFlags();
667 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
668 emitByte(0xF3, OS);
669 if (Flags & X86::IP_HAS_REPEAT_NE)
670 emitByte(0xF2, OS);
671
672 // Emit the address size opcode prefix as needed.
673 bool NeedAddressOverride;
674 uint64_t AdSize = TSFlags & X86II::AdSizeMask;
675 if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
676 (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
677 (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
678 NeedAddressOverride = true;
679 } else if (MemoryOperand < 0) {
680 NeedAddressOverride = false;
681 } else if (STI.hasFeature(X86::Mode64Bit)) {
682 assert(!is16BitMemOperand(MI, MemoryOperand, STI));
683 NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
684 } else if (STI.hasFeature(X86::Mode32Bit)) {
685 assert(!is64BitMemOperand(MI, MemoryOperand));
686 NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
687 } else {
688 assert(STI.hasFeature(X86::Mode16Bit));
689 assert(!is64BitMemOperand(MI, MemoryOperand));
690 NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
691 }
692
693 if (NeedAddressOverride)
694 emitByte(0x67, OS);
695
696 // Encoding type for this instruction.
697 uint64_t Encoding = TSFlags & X86II::EncodingMask;
698 bool HasREX = false;
699 if (Encoding)
700 emitVEXOpcodePrefix(MemoryOperand, MI, OS);
701 else
702 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
703
704 uint64_t Form = TSFlags & X86II::FormMask;
705 switch (Form) {
706 default:
707 break;
708 case X86II::RawFrmDstSrc: {
709 unsigned siReg = MI.getOperand(1).getReg();
710 assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
711 (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
712 (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
713 "SI and DI register sizes do not match");
714 // Emit segment override opcode prefix as needed (not for %ds).
715 if (MI.getOperand(2).getReg() != X86::DS)
716 emitSegmentOverridePrefix(2, MI, OS);
717 // Emit AdSize prefix as needed.
718 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
719 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
720 emitByte(0x67, OS);
721 CurOp += 3; // Consume operands.
722 break;
723 }
724 case X86II::RawFrmSrc: {
725 unsigned siReg = MI.getOperand(0).getReg();
726 // Emit segment override opcode prefix as needed (not for %ds).
727 if (MI.getOperand(1).getReg() != X86::DS)
728 emitSegmentOverridePrefix(1, MI, OS);
729 // Emit AdSize prefix as needed.
730 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
731 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
732 emitByte(0x67, OS);
733 CurOp += 2; // Consume operands.
734 break;
735 }
736 case X86II::RawFrmDst: {
737 unsigned siReg = MI.getOperand(0).getReg();
738 // Emit AdSize prefix as needed.
739 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
740 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
741 emitByte(0x67, OS);
742 ++CurOp; // Consume operand.
743 break;
744 }
745 case X86II::RawFrmMemOffs: {
746 // Emit segment override opcode prefix as needed.
747 emitSegmentOverridePrefix(1, MI, OS);
748 break;
749 }
750 }
751
752 return HasREX;
753 }
754
755 /// AVX instructions are encoded using a opcode prefix called VEX.
emitVEXOpcodePrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const756 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
757 raw_ostream &OS) const {
758 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
759 uint64_t TSFlags = Desc.TSFlags;
760
761 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
762
763 uint64_t Encoding = TSFlags & X86II::EncodingMask;
764 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
765 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
766 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
767
768 // VEX_R: opcode externsion equivalent to REX.R in
769 // 1's complement (inverted) form
770 //
771 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
772 // 0: Same as REX_R=1 (64 bit mode only)
773 //
774 uint8_t VEX_R = 0x1;
775 uint8_t EVEX_R2 = 0x1;
776
777 // VEX_X: equivalent to REX.X, only used when a
778 // register is used for index in SIB Byte.
779 //
780 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
781 // 0: Same as REX.X=1 (64-bit mode only)
782 uint8_t VEX_X = 0x1;
783
784 // VEX_B:
785 //
786 // 1: Same as REX_B=0 (ignored in 32-bit mode)
787 // 0: Same as REX_B=1 (64 bit mode only)
788 //
789 uint8_t VEX_B = 0x1;
790
791 // VEX_W: opcode specific (use like REX.W, or used for
792 // opcode extension, or ignored, depending on the opcode byte)
793 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
794
795 // VEX_5M (VEX m-mmmmm field):
796 //
797 // 0b00000: Reserved for future use
798 // 0b00001: implied 0F leading opcode
799 // 0b00010: implied 0F 38 leading opcode bytes
800 // 0b00011: implied 0F 3A leading opcode bytes
801 // 0b00100-0b11111: Reserved for future use
802 // 0b01000: XOP map select - 08h instructions with imm byte
803 // 0b01001: XOP map select - 09h instructions with no imm byte
804 // 0b01010: XOP map select - 0Ah instructions with imm dword
805 uint8_t VEX_5M;
806 switch (TSFlags & X86II::OpMapMask) {
807 default:
808 llvm_unreachable("Invalid prefix!");
809 case X86II::TB:
810 VEX_5M = 0x1;
811 break; // 0F
812 case X86II::T8:
813 VEX_5M = 0x2;
814 break; // 0F 38
815 case X86II::TA:
816 VEX_5M = 0x3;
817 break; // 0F 3A
818 case X86II::XOP8:
819 VEX_5M = 0x8;
820 break;
821 case X86II::XOP9:
822 VEX_5M = 0x9;
823 break;
824 case X86II::XOPA:
825 VEX_5M = 0xA;
826 break;
827 }
828
829 // VEX_4V (VEX vvvv field): a register specifier
830 // (in 1's complement form) or 1111 if unused.
831 uint8_t VEX_4V = 0xf;
832 uint8_t EVEX_V2 = 0x1;
833
834 // EVEX_L2/VEX_L (Vector Length):
835 //
836 // L2 L
837 // 0 0: scalar or 128-bit vector
838 // 0 1: 256-bit vector
839 // 1 0: 512-bit vector
840 //
841 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
842 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
843
844 // VEX_PP: opcode extension providing equivalent
845 // functionality of a SIMD prefix
846 //
847 // 0b00: None
848 // 0b01: 66
849 // 0b10: F3
850 // 0b11: F2
851 //
852 uint8_t VEX_PP = 0;
853 switch (TSFlags & X86II::OpPrefixMask) {
854 case X86II::PD:
855 VEX_PP = 0x1;
856 break; // 66
857 case X86II::XS:
858 VEX_PP = 0x2;
859 break; // F3
860 case X86II::XD:
861 VEX_PP = 0x3;
862 break; // F2
863 }
864
865 // EVEX_U
866 uint8_t EVEX_U = 1; // Always '1' so far
867
868 // EVEX_z
869 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
870
871 // EVEX_b
872 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
873
874 // EVEX_rc
875 uint8_t EVEX_rc = 0;
876
877 // EVEX_aaa
878 uint8_t EVEX_aaa = 0;
879
880 bool EncodeRC = false;
881
882 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
883 unsigned NumOps = Desc.getNumOperands();
884 unsigned CurOp = X86II::getOperandBias(Desc);
885
886 switch (TSFlags & X86II::FormMask) {
887 default:
888 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
889 case X86II::MRM_C0:
890 case X86II::RawFrm:
891 case X86II::PrefixByte:
892 break;
893 case X86II::MRMDestMemFSIB:
894 case X86II::MRMDestMem: {
895 // MRMDestMem instructions forms:
896 // MemAddr, src1(ModR/M)
897 // MemAddr, src1(VEX_4V), src2(ModR/M)
898 // MemAddr, src1(ModR/M), imm8
899 //
900 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
901 VEX_B = ~(BaseRegEnc >> 3) & 1;
902 unsigned IndexRegEnc =
903 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
904 VEX_X = ~(IndexRegEnc >> 3) & 1;
905 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
906 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
907
908 CurOp += X86::AddrNumOperands;
909
910 if (HasEVEX_K)
911 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
912
913 if (HasVEX_4V) {
914 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
915 VEX_4V = ~VRegEnc & 0xf;
916 EVEX_V2 = ~(VRegEnc >> 4) & 1;
917 }
918
919 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
920 VEX_R = ~(RegEnc >> 3) & 1;
921 EVEX_R2 = ~(RegEnc >> 4) & 1;
922 break;
923 }
924 case X86II::MRMSrcMemFSIB:
925 case X86II::MRMSrcMem: {
926 // MRMSrcMem instructions forms:
927 // src1(ModR/M), MemAddr
928 // src1(ModR/M), src2(VEX_4V), MemAddr
929 // src1(ModR/M), MemAddr, imm8
930 // src1(ModR/M), MemAddr, src2(Imm[7:4])
931 //
932 // FMA4:
933 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
934 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
935 VEX_R = ~(RegEnc >> 3) & 1;
936 EVEX_R2 = ~(RegEnc >> 4) & 1;
937
938 if (HasEVEX_K)
939 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
940
941 if (HasVEX_4V) {
942 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
943 VEX_4V = ~VRegEnc & 0xf;
944 EVEX_V2 = ~(VRegEnc >> 4) & 1;
945 }
946
947 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
948 VEX_B = ~(BaseRegEnc >> 3) & 1;
949 unsigned IndexRegEnc =
950 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
951 VEX_X = ~(IndexRegEnc >> 3) & 1;
952 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
953 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
954
955 break;
956 }
957 case X86II::MRMSrcMem4VOp3: {
958 // Instruction format for 4VOp3:
959 // src1(ModR/M), MemAddr, src3(VEX_4V)
960 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
961 VEX_R = ~(RegEnc >> 3) & 1;
962
963 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
964 VEX_B = ~(BaseRegEnc >> 3) & 1;
965 unsigned IndexRegEnc =
966 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
967 VEX_X = ~(IndexRegEnc >> 3) & 1;
968
969 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
970 break;
971 }
972 case X86II::MRMSrcMemOp4: {
973 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
974 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
975 VEX_R = ~(RegEnc >> 3) & 1;
976
977 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
978 VEX_4V = ~VRegEnc & 0xf;
979
980 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
981 VEX_B = ~(BaseRegEnc >> 3) & 1;
982 unsigned IndexRegEnc =
983 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
984 VEX_X = ~(IndexRegEnc >> 3) & 1;
985 break;
986 }
987 case X86II::MRM0m:
988 case X86II::MRM1m:
989 case X86II::MRM2m:
990 case X86II::MRM3m:
991 case X86II::MRM4m:
992 case X86II::MRM5m:
993 case X86II::MRM6m:
994 case X86II::MRM7m: {
995 // MRM[0-9]m instructions forms:
996 // MemAddr
997 // src1(VEX_4V), MemAddr
998 if (HasVEX_4V) {
999 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1000 VEX_4V = ~VRegEnc & 0xf;
1001 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1002 }
1003
1004 if (HasEVEX_K)
1005 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1006
1007 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
1008 VEX_B = ~(BaseRegEnc >> 3) & 1;
1009 unsigned IndexRegEnc =
1010 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
1011 VEX_X = ~(IndexRegEnc >> 3) & 1;
1012 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
1013 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
1014
1015 break;
1016 }
1017 case X86II::MRMSrcReg: {
1018 // MRMSrcReg instructions forms:
1019 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1020 // dst(ModR/M), src1(ModR/M)
1021 // dst(ModR/M), src1(ModR/M), imm8
1022 //
1023 // FMA4:
1024 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1025 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1026 VEX_R = ~(RegEnc >> 3) & 1;
1027 EVEX_R2 = ~(RegEnc >> 4) & 1;
1028
1029 if (HasEVEX_K)
1030 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1031
1032 if (HasVEX_4V) {
1033 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1034 VEX_4V = ~VRegEnc & 0xf;
1035 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1036 }
1037
1038 RegEnc = getX86RegEncoding(MI, CurOp++);
1039 VEX_B = ~(RegEnc >> 3) & 1;
1040 VEX_X = ~(RegEnc >> 4) & 1;
1041
1042 if (EVEX_b) {
1043 if (HasEVEX_RC) {
1044 unsigned RcOperand = NumOps - 1;
1045 assert(RcOperand >= CurOp);
1046 EVEX_rc = MI.getOperand(RcOperand).getImm();
1047 assert(EVEX_rc <= 3 && "Invalid rounding control!");
1048 }
1049 EncodeRC = true;
1050 }
1051 break;
1052 }
1053 case X86II::MRMSrcReg4VOp3: {
1054 // Instruction format for 4VOp3:
1055 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1056 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1057 VEX_R = ~(RegEnc >> 3) & 1;
1058
1059 RegEnc = getX86RegEncoding(MI, CurOp++);
1060 VEX_B = ~(RegEnc >> 3) & 1;
1061
1062 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1063 break;
1064 }
1065 case X86II::MRMSrcRegOp4: {
1066 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1067 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1068 VEX_R = ~(RegEnc >> 3) & 1;
1069
1070 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1071 VEX_4V = ~VRegEnc & 0xf;
1072
1073 // Skip second register source (encoded in Imm[7:4])
1074 ++CurOp;
1075
1076 RegEnc = getX86RegEncoding(MI, CurOp++);
1077 VEX_B = ~(RegEnc >> 3) & 1;
1078 VEX_X = ~(RegEnc >> 4) & 1;
1079 break;
1080 }
1081 case X86II::MRMDestReg: {
1082 // MRMDestReg instructions forms:
1083 // dst(ModR/M), src(ModR/M)
1084 // dst(ModR/M), src(ModR/M), imm8
1085 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1086 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1087 VEX_B = ~(RegEnc >> 3) & 1;
1088 VEX_X = ~(RegEnc >> 4) & 1;
1089
1090 if (HasEVEX_K)
1091 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1092
1093 if (HasVEX_4V) {
1094 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1095 VEX_4V = ~VRegEnc & 0xf;
1096 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1097 }
1098
1099 RegEnc = getX86RegEncoding(MI, CurOp++);
1100 VEX_R = ~(RegEnc >> 3) & 1;
1101 EVEX_R2 = ~(RegEnc >> 4) & 1;
1102 if (EVEX_b)
1103 EncodeRC = true;
1104 break;
1105 }
1106 case X86II::MRMr0: {
1107 // MRMr0 instructions forms:
1108 // 11:rrr:000
1109 // dst(ModR/M)
1110 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1111 VEX_R = ~(RegEnc >> 3) & 1;
1112 EVEX_R2 = ~(RegEnc >> 4) & 1;
1113 break;
1114 }
1115 case X86II::MRM0r:
1116 case X86II::MRM1r:
1117 case X86II::MRM2r:
1118 case X86II::MRM3r:
1119 case X86II::MRM4r:
1120 case X86II::MRM5r:
1121 case X86II::MRM6r:
1122 case X86II::MRM7r: {
1123 // MRM0r-MRM7r instructions forms:
1124 // dst(VEX_4V), src(ModR/M), imm8
1125 if (HasVEX_4V) {
1126 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1127 VEX_4V = ~VRegEnc & 0xf;
1128 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1129 }
1130 if (HasEVEX_K)
1131 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1132
1133 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1134 VEX_B = ~(RegEnc >> 3) & 1;
1135 VEX_X = ~(RegEnc >> 4) & 1;
1136 break;
1137 }
1138 }
1139
1140 if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1141 // VEX opcode prefix can have 2 or 3 bytes
1142 //
1143 // 3 bytes:
1144 // +-----+ +--------------+ +-------------------+
1145 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1146 // +-----+ +--------------+ +-------------------+
1147 // 2 bytes:
1148 // +-----+ +-------------------+
1149 // | C5h | | R | vvvv | L | pp |
1150 // +-----+ +-------------------+
1151 //
1152 // XOP uses a similar prefix:
1153 // +-----+ +--------------+ +-------------------+
1154 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1155 // +-----+ +--------------+ +-------------------+
1156 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1157
1158 // Can we use the 2 byte VEX prefix?
1159 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1160 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1161 emitByte(0xC5, OS);
1162 emitByte(LastByte | (VEX_R << 7), OS);
1163 return;
1164 }
1165
1166 // 3 byte VEX prefix
1167 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1168 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1169 emitByte(LastByte | (VEX_W << 7), OS);
1170 } else {
1171 assert(Encoding == X86II::EVEX && "unknown encoding!");
1172 // EVEX opcode prefix can have 4 bytes
1173 //
1174 // +-----+ +--------------+ +-------------------+ +------------------------+
1175 // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1176 // +-----+ +--------------+ +-------------------+ +------------------------+
1177 assert((VEX_5M & 0x3) == VEX_5M &&
1178 "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
1179
1180 emitByte(0x62, OS);
1181 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1182 VEX_5M,
1183 OS);
1184 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1185 if (EncodeRC)
1186 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1187 EVEX_aaa,
1188 OS);
1189 else
1190 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1191 (EVEX_V2 << 3) | EVEX_aaa,
1192 OS);
1193 }
1194 }
1195
1196 /// Emit REX prefix which specifies
1197 /// 1) 64-bit instructions,
1198 /// 2) non-default operand size, and
1199 /// 3) use of X86-64 extended registers.
1200 ///
1201 /// \returns true if REX prefix is used, otherwise returns false.
emitREXPrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const1202 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1203 raw_ostream &OS) const {
1204 uint8_t REX = [&, MemOperand]() {
1205 uint8_t REX = 0;
1206 bool UsesHighByteReg = false;
1207
1208 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1209 uint64_t TSFlags = Desc.TSFlags;
1210
1211 if (TSFlags & X86II::REX_W)
1212 REX |= 1 << 3; // set REX.W
1213
1214 if (MI.getNumOperands() == 0)
1215 return REX;
1216
1217 unsigned NumOps = MI.getNumOperands();
1218 unsigned CurOp = X86II::getOperandBias(Desc);
1219
1220 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1221 for (unsigned i = CurOp; i != NumOps; ++i) {
1222 const MCOperand &MO = MI.getOperand(i);
1223 if (!MO.isReg())
1224 continue;
1225 unsigned Reg = MO.getReg();
1226 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1227 UsesHighByteReg = true;
1228 if (X86II::isX86_64NonExtLowByteReg(Reg))
1229 // FIXME: The caller of determineREXPrefix slaps this prefix onto
1230 // anything that returns non-zero.
1231 REX |= 0x40; // REX fixed encoding prefix
1232 }
1233
1234 switch (TSFlags & X86II::FormMask) {
1235 case X86II::AddRegFrm:
1236 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1237 break;
1238 case X86II::MRMSrcReg:
1239 case X86II::MRMSrcRegCC:
1240 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1241 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1242 break;
1243 case X86II::MRMSrcMem:
1244 case X86II::MRMSrcMemCC:
1245 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1246 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1247 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1248 CurOp += X86::AddrNumOperands;
1249 break;
1250 case X86II::MRMDestReg:
1251 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1252 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1253 break;
1254 case X86II::MRMDestMem:
1255 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1256 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1257 CurOp += X86::AddrNumOperands;
1258 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1259 break;
1260 case X86II::MRMXmCC:
1261 case X86II::MRMXm:
1262 case X86II::MRM0m:
1263 case X86II::MRM1m:
1264 case X86II::MRM2m:
1265 case X86II::MRM3m:
1266 case X86II::MRM4m:
1267 case X86II::MRM5m:
1268 case X86II::MRM6m:
1269 case X86II::MRM7m:
1270 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1271 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1272 break;
1273 case X86II::MRMXrCC:
1274 case X86II::MRMXr:
1275 case X86II::MRM0r:
1276 case X86II::MRM1r:
1277 case X86II::MRM2r:
1278 case X86II::MRM3r:
1279 case X86II::MRM4r:
1280 case X86II::MRM5r:
1281 case X86II::MRM6r:
1282 case X86II::MRM7r:
1283 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1284 break;
1285 case X86II::MRMr0:
1286 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1287 break;
1288 case X86II::MRMDestMemFSIB:
1289 llvm_unreachable("FSIB format never need REX prefix!");
1290 }
1291 if (REX && UsesHighByteReg)
1292 report_fatal_error(
1293 "Cannot encode high byte register in REX-prefixed instruction");
1294 return REX;
1295 }();
1296
1297 if (!REX)
1298 return false;
1299
1300 emitByte(0x40 | REX, OS);
1301 return true;
1302 }
1303
1304 /// Emit segment override opcode prefix as needed.
emitSegmentOverridePrefix(unsigned SegOperand,const MCInst & MI,raw_ostream & OS) const1305 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1306 const MCInst &MI,
1307 raw_ostream &OS) const {
1308 // Check for explicit segment override on memory operand.
1309 if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1310 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1311 }
1312
1313 /// Emit all instruction prefixes prior to the opcode.
1314 ///
1315 /// \param MemOperand the operand # of the start of a memory operand if present.
1316 /// If not present, it is -1.
1317 ///
1318 /// \returns true if REX prefix is used, otherwise returns false.
emitOpcodePrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1319 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1320 const MCSubtargetInfo &STI,
1321 raw_ostream &OS) const {
1322 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1323 uint64_t TSFlags = Desc.TSFlags;
1324
1325 // Emit the operand size opcode prefix as needed.
1326 if ((TSFlags & X86II::OpSizeMask) ==
1327 (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1328 emitByte(0x66, OS);
1329
1330 // Emit the LOCK opcode prefix.
1331 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1332 emitByte(0xF0, OS);
1333
1334 // Emit the NOTRACK opcode prefix.
1335 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1336 emitByte(0x3E, OS);
1337
1338 switch (TSFlags & X86II::OpPrefixMask) {
1339 case X86II::PD: // 66
1340 emitByte(0x66, OS);
1341 break;
1342 case X86II::XS: // F3
1343 emitByte(0xF3, OS);
1344 break;
1345 case X86II::XD: // F2
1346 emitByte(0xF2, OS);
1347 break;
1348 }
1349
1350 // Handle REX prefix.
1351 assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1352 "REX.W requires 64bit mode.");
1353 bool HasREX = STI.hasFeature(X86::Mode64Bit)
1354 ? emitREXPrefix(MemOperand, MI, OS)
1355 : false;
1356
1357 // 0x0F escape code must be emitted just before the opcode.
1358 switch (TSFlags & X86II::OpMapMask) {
1359 case X86II::TB: // Two-byte opcode map
1360 case X86II::T8: // 0F 38
1361 case X86II::TA: // 0F 3A
1362 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1363 emitByte(0x0F, OS);
1364 break;
1365 }
1366
1367 switch (TSFlags & X86II::OpMapMask) {
1368 case X86II::T8: // 0F 38
1369 emitByte(0x38, OS);
1370 break;
1371 case X86II::TA: // 0F 3A
1372 emitByte(0x3A, OS);
1373 break;
1374 }
1375
1376 return HasREX;
1377 }
1378
emitPrefix(const MCInst & MI,raw_ostream & OS,const MCSubtargetInfo & STI) const1379 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1380 const MCSubtargetInfo &STI) const {
1381 unsigned Opcode = MI.getOpcode();
1382 const MCInstrDesc &Desc = MCII.get(Opcode);
1383 uint64_t TSFlags = Desc.TSFlags;
1384
1385 // Pseudo instructions don't get encoded.
1386 if (X86II::isPseudo(TSFlags))
1387 return;
1388
1389 unsigned CurOp = X86II::getOperandBias(Desc);
1390
1391 emitPrefixImpl(CurOp, MI, STI, OS);
1392 }
1393
encodeInstruction(const MCInst & MI,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI) const1394 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1395 SmallVectorImpl<MCFixup> &Fixups,
1396 const MCSubtargetInfo &STI) const {
1397 unsigned Opcode = MI.getOpcode();
1398 const MCInstrDesc &Desc = MCII.get(Opcode);
1399 uint64_t TSFlags = Desc.TSFlags;
1400
1401 // Pseudo instructions don't get encoded.
1402 if (X86II::isPseudo(TSFlags))
1403 return;
1404
1405 unsigned NumOps = Desc.getNumOperands();
1406 unsigned CurOp = X86II::getOperandBias(Desc);
1407
1408 uint64_t StartByte = OS.tell();
1409
1410 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1411
1412 // It uses the VEX.VVVV field?
1413 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1414 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1415
1416 // It uses the EVEX.aaa field?
1417 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1418 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1419
1420 // Used if a register is encoded in 7:4 of immediate.
1421 unsigned I8RegNum = 0;
1422
1423 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1424
1425 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1426 BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1427
1428 unsigned OpcodeOffset = 0;
1429
1430 uint64_t Form = TSFlags & X86II::FormMask;
1431 switch (Form) {
1432 default:
1433 errs() << "FORM: " << Form << "\n";
1434 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1435 case X86II::Pseudo:
1436 llvm_unreachable("Pseudo instruction shouldn't be emitted");
1437 case X86II::RawFrmDstSrc:
1438 case X86II::RawFrmSrc:
1439 case X86II::RawFrmDst:
1440 case X86II::PrefixByte:
1441 emitByte(BaseOpcode, OS);
1442 break;
1443 case X86II::AddCCFrm: {
1444 // This will be added to the opcode in the fallthrough.
1445 OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1446 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1447 --NumOps; // Drop the operand from the end.
1448 LLVM_FALLTHROUGH;
1449 case X86II::RawFrm:
1450 emitByte(BaseOpcode + OpcodeOffset, OS);
1451
1452 if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1453 break;
1454
1455 const MCOperand &Op = MI.getOperand(CurOp++);
1456 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1457 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1458 Fixups);
1459 break;
1460 }
1461 case X86II::RawFrmMemOffs:
1462 emitByte(BaseOpcode, OS);
1463 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1464 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1465 StartByte, OS, Fixups);
1466 ++CurOp; // skip segment operand
1467 break;
1468 case X86II::RawFrmImm8:
1469 emitByte(BaseOpcode, OS);
1470 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1471 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1472 StartByte, OS, Fixups);
1473 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1474 OS, Fixups);
1475 break;
1476 case X86II::RawFrmImm16:
1477 emitByte(BaseOpcode, OS);
1478 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1479 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1480 StartByte, OS, Fixups);
1481 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1482 OS, Fixups);
1483 break;
1484
1485 case X86II::AddRegFrm:
1486 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1487 break;
1488
1489 case X86II::MRMDestReg: {
1490 emitByte(BaseOpcode, OS);
1491 unsigned SrcRegNum = CurOp + 1;
1492
1493 if (HasEVEX_K) // Skip writemask
1494 ++SrcRegNum;
1495
1496 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1497 ++SrcRegNum;
1498
1499 emitRegModRMByte(MI.getOperand(CurOp),
1500 getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1501 CurOp = SrcRegNum + 1;
1502 break;
1503 }
1504 case X86II::MRMDestMemFSIB:
1505 case X86II::MRMDestMem: {
1506 emitByte(BaseOpcode, OS);
1507 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1508
1509 if (HasEVEX_K) // Skip writemask
1510 ++SrcRegNum;
1511
1512 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1513 ++SrcRegNum;
1514
1515 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1516 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1517 HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1518 CurOp = SrcRegNum + 1;
1519 break;
1520 }
1521 case X86II::MRMSrcReg: {
1522 emitByte(BaseOpcode, OS);
1523 unsigned SrcRegNum = CurOp + 1;
1524
1525 if (HasEVEX_K) // Skip writemask
1526 ++SrcRegNum;
1527
1528 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1529 ++SrcRegNum;
1530
1531 emitRegModRMByte(MI.getOperand(SrcRegNum),
1532 getX86RegNum(MI.getOperand(CurOp)), OS);
1533 CurOp = SrcRegNum + 1;
1534 if (HasVEX_I8Reg)
1535 I8RegNum = getX86RegEncoding(MI, CurOp++);
1536 // do not count the rounding control operand
1537 if (HasEVEX_RC)
1538 --NumOps;
1539 break;
1540 }
1541 case X86II::MRMSrcReg4VOp3: {
1542 emitByte(BaseOpcode, OS);
1543 unsigned SrcRegNum = CurOp + 1;
1544
1545 emitRegModRMByte(MI.getOperand(SrcRegNum),
1546 getX86RegNum(MI.getOperand(CurOp)), OS);
1547 CurOp = SrcRegNum + 1;
1548 ++CurOp; // Encoded in VEX.VVVV
1549 break;
1550 }
1551 case X86II::MRMSrcRegOp4: {
1552 emitByte(BaseOpcode, OS);
1553 unsigned SrcRegNum = CurOp + 1;
1554
1555 // Skip 1st src (which is encoded in VEX_VVVV)
1556 ++SrcRegNum;
1557
1558 // Capture 2nd src (which is encoded in Imm[7:4])
1559 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1560 I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1561
1562 emitRegModRMByte(MI.getOperand(SrcRegNum),
1563 getX86RegNum(MI.getOperand(CurOp)), OS);
1564 CurOp = SrcRegNum + 1;
1565 break;
1566 }
1567 case X86II::MRMSrcRegCC: {
1568 unsigned FirstOp = CurOp++;
1569 unsigned SecondOp = CurOp++;
1570
1571 unsigned CC = MI.getOperand(CurOp++).getImm();
1572 emitByte(BaseOpcode + CC, OS);
1573
1574 emitRegModRMByte(MI.getOperand(SecondOp),
1575 getX86RegNum(MI.getOperand(FirstOp)), OS);
1576 break;
1577 }
1578 case X86II::MRMSrcMemFSIB:
1579 case X86II::MRMSrcMem: {
1580 unsigned FirstMemOp = CurOp + 1;
1581
1582 if (HasEVEX_K) // Skip writemask
1583 ++FirstMemOp;
1584
1585 if (HasVEX_4V)
1586 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1587
1588 emitByte(BaseOpcode, OS);
1589
1590 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1591 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1592 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1593 CurOp = FirstMemOp + X86::AddrNumOperands;
1594 if (HasVEX_I8Reg)
1595 I8RegNum = getX86RegEncoding(MI, CurOp++);
1596 break;
1597 }
1598 case X86II::MRMSrcMem4VOp3: {
1599 unsigned FirstMemOp = CurOp + 1;
1600
1601 emitByte(BaseOpcode, OS);
1602
1603 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1604 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1605 CurOp = FirstMemOp + X86::AddrNumOperands;
1606 ++CurOp; // Encoded in VEX.VVVV.
1607 break;
1608 }
1609 case X86II::MRMSrcMemOp4: {
1610 unsigned FirstMemOp = CurOp + 1;
1611
1612 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1613
1614 // Capture second register source (encoded in Imm[7:4])
1615 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1616 I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1617
1618 emitByte(BaseOpcode, OS);
1619
1620 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1621 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1622 CurOp = FirstMemOp + X86::AddrNumOperands;
1623 break;
1624 }
1625 case X86II::MRMSrcMemCC: {
1626 unsigned RegOp = CurOp++;
1627 unsigned FirstMemOp = CurOp;
1628 CurOp = FirstMemOp + X86::AddrNumOperands;
1629
1630 unsigned CC = MI.getOperand(CurOp++).getImm();
1631 emitByte(BaseOpcode + CC, OS);
1632
1633 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1634 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1635 break;
1636 }
1637
1638 case X86II::MRMXrCC: {
1639 unsigned RegOp = CurOp++;
1640
1641 unsigned CC = MI.getOperand(CurOp++).getImm();
1642 emitByte(BaseOpcode + CC, OS);
1643 emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1644 break;
1645 }
1646
1647 case X86II::MRMXr:
1648 case X86II::MRM0r:
1649 case X86II::MRM1r:
1650 case X86II::MRM2r:
1651 case X86II::MRM3r:
1652 case X86II::MRM4r:
1653 case X86II::MRM5r:
1654 case X86II::MRM6r:
1655 case X86II::MRM7r:
1656 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1657 ++CurOp;
1658 if (HasEVEX_K) // Skip writemask
1659 ++CurOp;
1660 emitByte(BaseOpcode, OS);
1661 emitRegModRMByte(MI.getOperand(CurOp++),
1662 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1663 break;
1664 case X86II::MRMr0:
1665 emitByte(BaseOpcode, OS);
1666 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1667 break;
1668
1669 case X86II::MRMXmCC: {
1670 unsigned FirstMemOp = CurOp;
1671 CurOp = FirstMemOp + X86::AddrNumOperands;
1672
1673 unsigned CC = MI.getOperand(CurOp++).getImm();
1674 emitByte(BaseOpcode + CC, OS);
1675
1676 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1677 STI);
1678 break;
1679 }
1680
1681 case X86II::MRMXm:
1682 case X86II::MRM0m:
1683 case X86II::MRM1m:
1684 case X86II::MRM2m:
1685 case X86II::MRM3m:
1686 case X86II::MRM4m:
1687 case X86II::MRM5m:
1688 case X86II::MRM6m:
1689 case X86II::MRM7m:
1690 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1691 ++CurOp;
1692 if (HasEVEX_K) // Skip writemask
1693 ++CurOp;
1694 emitByte(BaseOpcode, OS);
1695 emitMemModRMByte(MI, CurOp,
1696 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1697 HasREX, StartByte, OS, Fixups, STI);
1698 CurOp += X86::AddrNumOperands;
1699 break;
1700
1701 case X86II::MRM0X:
1702 case X86II::MRM1X:
1703 case X86II::MRM2X:
1704 case X86II::MRM3X:
1705 case X86II::MRM4X:
1706 case X86II::MRM5X:
1707 case X86II::MRM6X:
1708 case X86II::MRM7X:
1709 emitByte(BaseOpcode, OS);
1710 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1711 break;
1712
1713 case X86II::MRM_C0:
1714 case X86II::MRM_C1:
1715 case X86II::MRM_C2:
1716 case X86II::MRM_C3:
1717 case X86II::MRM_C4:
1718 case X86II::MRM_C5:
1719 case X86II::MRM_C6:
1720 case X86II::MRM_C7:
1721 case X86II::MRM_C8:
1722 case X86II::MRM_C9:
1723 case X86II::MRM_CA:
1724 case X86II::MRM_CB:
1725 case X86II::MRM_CC:
1726 case X86II::MRM_CD:
1727 case X86II::MRM_CE:
1728 case X86II::MRM_CF:
1729 case X86II::MRM_D0:
1730 case X86II::MRM_D1:
1731 case X86II::MRM_D2:
1732 case X86II::MRM_D3:
1733 case X86II::MRM_D4:
1734 case X86II::MRM_D5:
1735 case X86II::MRM_D6:
1736 case X86II::MRM_D7:
1737 case X86II::MRM_D8:
1738 case X86II::MRM_D9:
1739 case X86II::MRM_DA:
1740 case X86II::MRM_DB:
1741 case X86II::MRM_DC:
1742 case X86II::MRM_DD:
1743 case X86II::MRM_DE:
1744 case X86II::MRM_DF:
1745 case X86II::MRM_E0:
1746 case X86II::MRM_E1:
1747 case X86II::MRM_E2:
1748 case X86II::MRM_E3:
1749 case X86II::MRM_E4:
1750 case X86II::MRM_E5:
1751 case X86II::MRM_E6:
1752 case X86II::MRM_E7:
1753 case X86II::MRM_E8:
1754 case X86II::MRM_E9:
1755 case X86II::MRM_EA:
1756 case X86II::MRM_EB:
1757 case X86II::MRM_EC:
1758 case X86II::MRM_ED:
1759 case X86II::MRM_EE:
1760 case X86II::MRM_EF:
1761 case X86II::MRM_F0:
1762 case X86II::MRM_F1:
1763 case X86II::MRM_F2:
1764 case X86II::MRM_F3:
1765 case X86II::MRM_F4:
1766 case X86II::MRM_F5:
1767 case X86II::MRM_F6:
1768 case X86II::MRM_F7:
1769 case X86II::MRM_F8:
1770 case X86II::MRM_F9:
1771 case X86II::MRM_FA:
1772 case X86II::MRM_FB:
1773 case X86II::MRM_FC:
1774 case X86II::MRM_FD:
1775 case X86II::MRM_FE:
1776 case X86II::MRM_FF:
1777 emitByte(BaseOpcode, OS);
1778 emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1779 break;
1780 }
1781
1782 if (HasVEX_I8Reg) {
1783 // The last source register of a 4 operand instruction in AVX is encoded
1784 // in bits[7:4] of a immediate byte.
1785 assert(I8RegNum < 16 && "Register encoding out of range");
1786 I8RegNum <<= 4;
1787 if (CurOp != NumOps) {
1788 unsigned Val = MI.getOperand(CurOp++).getImm();
1789 assert(Val < 16 && "Immediate operand value out of range");
1790 I8RegNum |= Val;
1791 }
1792 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1793 StartByte, OS, Fixups);
1794 } else {
1795 // If there is a remaining operand, it must be a trailing immediate. Emit it
1796 // according to the right size for the instruction. Some instructions
1797 // (SSE4a extrq and insertq) have two trailing immediates.
1798 while (CurOp != NumOps && NumOps - CurOp <= 2) {
1799 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1800 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1801 StartByte, OS, Fixups);
1802 }
1803 }
1804
1805 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1806 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1807
1808 assert(OS.tell() - StartByte <= 15 &&
1809 "The size of instruction must be no longer than 15.");
1810 #ifndef NDEBUG
1811 // FIXME: Verify.
1812 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1813 errs() << "Cannot encode all operands of: ";
1814 MI.dump();
1815 errs() << '\n';
1816 abort();
1817 }
1818 #endif
1819 }
1820
createX86MCCodeEmitter(const MCInstrInfo & MCII,const MCRegisterInfo & MRI,MCContext & Ctx)1821 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1822 const MCRegisterInfo &MRI,
1823 MCContext &Ctx) {
1824 return new X86MCCodeEmitter(MCII, Ctx);
1825 }
1826